From ed5461daa150b037e36b8202381da1ef85d6b16b Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Tue, 15 Jul 2025 03:25:03 +0530 Subject: [PATCH 001/169] drm/xe: Don't fail probe on unsupported mailbox command If the device is running older pcode firmware, it is possible that newer mailbox commands are not supported by it. The sysfs attributes aren't useful in that case, but we shouldn't fail driver probe because of it. As of now, it is unknown if we can distinguish unsupported commands before attempting them. But until we figure out a way to do that, fix the regressions. v2: Add debug message (Lucas) Fixes: cdc36b66cd41 ("drm/xe: Expose fan control and voltage regulator version") Signed-off-by: Raag Jadav Tested-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250714215503.2897748-1-raag.jadav@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device_sysfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c index e5fd0cd537bc..bd9015761aa0 100644 --- a/drivers/gpu/drm/xe/xe_device_sysfs.c +++ b/drivers/gpu/drm/xe/xe_device_sysfs.c @@ -160,8 +160,13 @@ static int late_bind_create_files(struct device *dev) ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), &cap, NULL); - if (ret) + if (ret) { + if (ret == -ENXIO) { + drm_dbg(&xe->drm, "Late binding not supported by firmware\n"); + ret = 0; + } goto out; + } if (REG_FIELD_GET(V1_FAN_SUPPORTED, cap)) { ret = sysfs_create_file(&dev->kobj, &dev_attr_lb_fan_control_version.attr); From 308dc9b27874d0e8a0258869b9e681b0fdd2e579 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Tue, 15 Jul 2025 11:14:22 -0700 Subject: [PATCH 002/169] drm/xe/oa: Fix static checker warning about null gt There is a static checker warning that gt returned by xe_device_get_gt can be NULL and that is being dereferenced. Use xe_root_mmio_gt instead, which is equivalent and cannot return a NULL gt 0. Fixes: 10d42ef34bce ("drm/xe/oa: Assign hwe for OAM_SAG") Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://lore.kernel.org/r/20250715181422.2807624-1-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index d991fbd90f20..5729e7d3e335 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1941,7 +1941,7 @@ static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param) /* If not provided, OA unit defaults to OA unit 0 as per uapi */ if (!param->oa_unit) - param->oa_unit = &xe_device_get_gt(oa->xe, 0)->oa.oa_unit[0]; + param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; /* When we have an exec_q, get hwe from the exec_q */ if (param->exec_q) { From 2f264d58cc805a3cefc6b98097f90fbc388136ef Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 15 Jul 2025 17:59:44 -0500 Subject: [PATCH 003/169] drm/xe: Fix a NULL vs IS_ERR() bug in xe_i2c_register_adapter() The fwnode_create_software_node() function returns error pointers. It never returns NULL. Update the checks to match. Fixes: f0e53aadd702 ("drm/xe: Support for I2C attached MCUs") Signed-off-by: Dan Carpenter Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/65825d00-81ab-4665-af51-4fff6786a250@sabinyo.mountain Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_i2c.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index db9c0340be5c..1f19718db559 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -96,8 +96,8 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) int ret; fwnode = fwnode_create_software_node(xe_i2c_adapter_properties, NULL); - if (!fwnode) - return -ENOMEM; + if (IS_ERR(fwnode)) + return PTR_ERR(fwnode); /* * Not using platform_device_register_full() here because we don't have From bf81505f7dba467c17097315864790f9f7dca08a Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 14 Jul 2025 21:36:45 +0200 Subject: [PATCH 004/169] drm/xe: Move debugfs GT attributes under tile directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While in sysfs we are correctly trying to reflect the hardware architecture and we expose GT attributes in per tile hierarchy, in debugfs we expose GT attributes at flat level, without tiles. Create debugfs directories to represent each tile and move GT attributes under matching parent tile directory. To not break existing debugfs tools, create symlink under old location: /sys/kernel/debug/dri/0000:00:02.0/ ├── ... ├── gt0 -> tile0/gt0 ├── gt1 -> tile0/gt1 ├── tile0 │   ├── gt0 │   │   ├── ... │   ├── gt1 │   │   ├── ... Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Cc: Rodrigo Vivi Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250714193645.763-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 22 ++++++++++++++++++++++ drivers/gpu/drm/xe/xe_device_types.h | 3 +++ drivers/gpu/drm/xe/xe_gt_debugfs.c | 14 +++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 26e9d146ccbf..129186f4193b 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -247,14 +247,33 @@ static const struct file_operations atomic_svm_timeslice_ms_fops = { .write = atomic_svm_timeslice_ms_set, }; +static void create_tile_debugfs(struct xe_tile *tile, struct dentry *root) +{ + char name[8]; + + snprintf(name, sizeof(name), "tile%u", tile->id); + tile->debugfs = debugfs_create_dir(name, root); + if (IS_ERR(tile->debugfs)) + return; + + /* + * Store the xe_tile pointer as private data of the tile/ directory + * node so other tile specific attributes under that directory may + * refer to it by looking at its parent node private data. + */ + tile->debugfs->d_inode->i_private = tile; +} + void xe_debugfs_register(struct xe_device *xe) { struct ttm_device *bdev = &xe->ttm; struct drm_minor *minor = xe->drm.primary; struct dentry *root = minor->debugfs_root; struct ttm_resource_manager *man; + struct xe_tile *tile; struct xe_gt *gt; u32 mem_type; + u8 tile_id; u8 id; drm_debugfs_create_files(debugfs_list, @@ -288,6 +307,9 @@ void xe_debugfs_register(struct xe_device *xe) if (man) ttm_resource_manager_create_debugfs(man, root, "stolen_mm"); + for_each_tile(tile, xe, tile_id) + create_tile_debugfs(tile, root); + for_each_gt(gt, xe, id) xe_gt_debugfs_register(gt); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index d4d2c6854790..98ce1255f1d0 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -255,6 +255,9 @@ struct xe_tile { /** @sysfs: sysfs' kobj used by xe_tile_sysfs */ struct kobject *sysfs; + + /** @debugfs: debugfs directory associated with this tile */ + struct dentry *debugfs; }; /** diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 848618acdca8..061a81f82e76 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -388,13 +388,18 @@ void xe_gt_debugfs_register(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); struct drm_minor *minor = gt_to_xe(gt)->drm.primary; + struct dentry *parent = gt->tile->debugfs; struct dentry *root; + char symlink[16]; char name[8]; xe_gt_assert(gt, minor->debugfs_root); + if (IS_ERR(parent)) + return; + snprintf(name, sizeof(name), "gt%d", gt->info.id); - root = debugfs_create_dir(name, minor->debugfs_root); + root = debugfs_create_dir(name, parent); if (IS_ERR(root)) { drm_warn(&xe->drm, "Create GT directory failed"); return; @@ -426,4 +431,11 @@ void xe_gt_debugfs_register(struct xe_gt *gt) xe_gt_sriov_pf_debugfs_register(gt, root); else if (IS_SRIOV_VF(xe)) xe_gt_sriov_vf_debugfs_register(gt, root); + + /* + * Backwards compatibility only: create a link for the legacy clients + * who may expect gt/ directory at the root level, not the tile level. + */ + snprintf(symlink, sizeof(symlink), "tile%u/%s", gt->tile->id, name); + debugfs_create_symlink(name, minor->debugfs_root, symlink); } From 922ae875230be91c7f05f2aa90d176b6693e2601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 14 Jul 2025 20:48:14 +0200 Subject: [PATCH 005/169] drm/xe: Use devm_ioremap_wc for VRAM mapping and drop manual unmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's replace the manual call to ioremap_wc function with devm_ioremap_wc function, ensuring that VRAM mappings are automatically released when the driver is detached. Since devm_ioremap_wc registers the mapping with the device's managed resources, the explicit iounmap call in vram_fini is no longer needed, so let's remove it. Signed-off-by: Piotr Piórkowski Suggested-by: Matthew Auld Reviewed-by: Matthew Auld Acked-by: Matthew Brost Link: https://lore.kernel.org/r/20250714184818.89201-2-piotr.piorkowski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_vram.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index e421a74fb87c..3a4c84e9efc6 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -156,7 +156,8 @@ static int determine_lmem_bar_size(struct xe_device *xe) xe->mem.vram.dpa_base = 0; /* set up a map to the total memory area. */ - xe->mem.vram.mapping = ioremap_wc(xe->mem.vram.io_start, xe->mem.vram.io_size); + xe->mem.vram.mapping = devm_ioremap_wc(&pdev->dev, xe->mem.vram.io_start, + xe->mem.vram.io_size); return 0; } @@ -278,9 +279,6 @@ static void vram_fini(void *arg) struct xe_tile *tile; int id; - if (xe->mem.vram.mapping) - iounmap(xe->mem.vram.mapping); - xe->mem.vram.mapping = NULL; for_each_tile(tile, xe, id) From f92cfd72d9a650f90260c54accd840c6500c4c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 14 Jul 2025 20:48:15 +0200 Subject: [PATCH 006/169] drm/xe: Use dynamic allocation for tile and device VRAM region structures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In future platforms, we will need to represent the device and tile VRAM regions in a more dynamic way, so let's abandon the static allocation of these structures and start use a dynamic allocation. v2: - Add a helpers for accessing fields of the xe_vram_region structure v3: - Add missing EXPORT_SYMBOL_IF_KUNIT for xe_vram_region_actual_physical_size Signed-off-by: Piotr Piórkowski Cc: Stuart Summers Cc: Matthew Auld Cc: Satyanarayana K V P Reviewed-by: Satyanarayana K V P Acked-by: Matthew Brost Link: https://lore.kernel.org/r/20250714184818.89201-3-piotr.piorkowski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/display/xe_fb_pin.c | 4 +- drivers/gpu/drm/xe/display/xe_plane_initial.c | 2 +- drivers/gpu/drm/xe/xe_assert.h | 4 +- drivers/gpu/drm/xe/xe_device.c | 19 +++ drivers/gpu/drm/xe/xe_device_types.h | 6 +- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 2 +- drivers/gpu/drm/xe/xe_migrate.c | 25 ++-- drivers/gpu/drm/xe/xe_pci.c | 6 + drivers/gpu/drm/xe/xe_query.c | 2 +- drivers/gpu/drm/xe/xe_svm.c | 24 +--- drivers/gpu/drm/xe/xe_tile.c | 34 ++++- drivers/gpu/drm/xe/xe_tile.h | 4 +- drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c | 10 +- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 7 +- drivers/gpu/drm/xe/xe_vram.c | 121 +++++++++++++----- drivers/gpu/drm/xe/xe_vram.h | 9 ++ 16 files changed, 202 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 405967df2c29..a1b203f40390 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -289,7 +289,7 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { - struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; /* * If we need to able to access the clear-color value stored in @@ -297,7 +297,7 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, * accessible. This is important on small-bar systems where * only some subset of VRAM is CPU accessible. */ - if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { + if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) { ret = -EINVAL; goto err; } diff --git a/drivers/gpu/drm/xe/display/xe_plane_initial.c b/drivers/gpu/drm/xe/display/xe_plane_initial.c index af47ce34102c..30a341c96e76 100644 --- a/drivers/gpu/drm/xe/display/xe_plane_initial.c +++ b/drivers/gpu/drm/xe/display/xe_plane_initial.c @@ -103,7 +103,7 @@ initial_plane_bo(struct xe_device *xe, * We don't currently expect this to ever be placed in the * stolen portion. */ - if (phys_base >= tile0->mem.vram.usable_size) { + if (phys_base >= xe_vram_region_usable_size(tile0->mem.vram)) { drm_err(&xe->drm, "Initial plane programming using invalid range, phys_base=%pa\n", &phys_base); diff --git a/drivers/gpu/drm/xe/xe_assert.h b/drivers/gpu/drm/xe/xe_assert.h index 68fe70ce2be3..a818eaa05b7d 100644 --- a/drivers/gpu/drm/xe/xe_assert.h +++ b/drivers/gpu/drm/xe/xe_assert.h @@ -12,6 +12,7 @@ #include "xe_gt_types.h" #include "xe_step.h" +#include "xe_vram.h" /** * DOC: Xe Asserts @@ -145,7 +146,8 @@ const struct xe_tile *__tile = (tile); \ char __buf[10] __maybe_unused; \ xe_assert_msg(tile_to_xe(__tile), condition, "tile: %u VRAM %s\n" msg, \ - __tile->id, ({ string_get_size(__tile->mem.vram.actual_physical_size, 1, \ + __tile->id, ({ string_get_size( \ + xe_vram_region_actual_physical_size(__tile->mem.vram), 1, \ STRING_UNITS_2, __buf, sizeof(__buf)); __buf; }), ## arg); \ }) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 6dc84e4ed281..df6f1790d6a6 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -687,6 +687,21 @@ static void sriov_update_device_info(struct xe_device *xe) } } +static int xe_device_vram_alloc(struct xe_device *xe) +{ + struct xe_vram_region *vram; + + if (!IS_DGFX(xe)) + return 0; + + vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); + if (!vram) + return -ENOMEM; + + xe->mem.vram = vram; + return 0; +} + /** * xe_device_probe_early: Device early probe * @xe: xe device instance @@ -734,6 +749,10 @@ int xe_device_probe_early(struct xe_device *xe) xe->wedged.mode = xe_modparam.wedged_mode; + err = xe_device_vram_alloc(xe); + if (err) + return err; + return 0; } ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 98ce1255f1d0..6299f4571c4c 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -77,6 +77,8 @@ struct xe_pxp; * device, such as HBM memory or CXL extension memory. */ struct xe_vram_region { + /** @tile: Back pointer to tile */ + struct xe_tile *tile; /** @io_start: IO start address of this VRAM instance */ resource_size_t io_start; /** @@ -216,7 +218,7 @@ struct xe_tile { * Although VRAM is associated with a specific tile, it can * still be accessed by all tiles' GTs. */ - struct xe_vram_region vram; + struct xe_vram_region *vram; /** @mem.ggtt: Global graphics translation table */ struct xe_ggtt *ggtt; @@ -415,7 +417,7 @@ struct xe_device { /** @mem: memory info for device */ struct { /** @mem.vram: VRAM info for device */ - struct xe_vram_region vram; + struct xe_vram_region *vram; /** @mem.sys_mgr: system TTM manager */ struct ttm_resource_manager sys_mgr; /** @mem.sys_mgr: system memory shrinker. */ diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 494909f74eb2..86889df04011 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -1604,7 +1604,7 @@ static u64 pf_query_free_lmem(struct xe_gt *gt) { struct xe_tile *tile = gt->tile; - return xe_ttm_vram_get_avail(&tile->mem.vram.ttm.manager); + return xe_ttm_vram_get_avail(&tile->mem.vram->ttm.manager); } static u64 pf_query_max_lmem(struct xe_gt *gt) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index ba1cff2e4cda..d6186ac530cb 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -34,6 +34,7 @@ #include "xe_sync.h" #include "xe_trace_bo.h" #include "xe_vm.h" +#include "xe_vram.h" /** * struct xe_migrate - migrate context. @@ -130,34 +131,36 @@ static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) u64 identity_offset = IDENTITY_OFFSET; if (GRAPHICS_VER(xe) >= 20 && is_comp_pte) - identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); + identity_offset += DIV_ROUND_UP_ULL(xe_vram_region_actual_physical_size + (xe->mem.vram), SZ_1G); - addr -= xe->mem.vram.dpa_base; + addr -= xe_vram_region_dpa_base(xe->mem.vram); return addr + (identity_offset << xe_pt_shift(2)); } static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) { + struct xe_vram_region *vram = xe->mem.vram; + resource_size_t dpa_base = xe_vram_region_dpa_base(vram); u64 pos, ofs, flags; u64 entry; /* XXX: Unclear if this should be usable_size? */ - u64 vram_limit = xe->mem.vram.actual_physical_size + - xe->mem.vram.dpa_base; + u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base; u32 level = 2; ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, true, 0); - xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M)); + xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M)); /* * Use 1GB pages when possible, last chunk always use 2M * pages as mixing reserved memory (stolen, WOCPM) with a single * mapping is not allowed on certain platforms. */ - for (pos = xe->mem.vram.dpa_base; pos < vram_limit; + for (pos = dpa_base; pos < vram_limit; pos += SZ_1G, ofs += 8) { if (pos + SZ_1G >= vram_limit) { entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs, @@ -307,11 +310,11 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, /* Identity map the entire vram at 256GiB offset */ if (IS_DGFX(xe)) { u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE; + resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram); xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, pat_index, pt30_ofs); - xe_assert(xe, xe->mem.vram.actual_physical_size <= - (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); + xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); /* * Identity map the entire vram for compressed pat_index for xe2+ @@ -320,11 +323,11 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; u64 vram_offset = IDENTITY_OFFSET + - DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); + DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; - xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE - - IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G); + xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET - + IDENTITY_OFFSET / 2) * SZ_1G); xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, comp_pat_index, pt31_ofs); } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 3c40ef426f0c..f64942737a0b 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -687,6 +687,8 @@ static int xe_info_init(struct xe_device *xe, * All of these together determine the overall GT count. */ for_each_tile(tile, xe, id) { + int err; + gt = tile->primary_gt; gt->info.type = XE_GT_TYPE_MAIN; gt->info.id = tile->id * xe->info.max_gt_per_tile; @@ -694,6 +696,10 @@ static int xe_info_init(struct xe_device *xe, gt->info.engine_mask = graphics_desc->hw_engine_mask; xe->info.gt_count++; + err = xe_tile_alloc_vram(tile); + if (err) + return err; + if (MEDIA_VER(xe) < 13 && media_desc) gt->info.engine_mask |= media_desc->hw_engine_mask; diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index d517ec9ddcbf..66ea3403afbe 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -337,7 +337,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->num_params = num_params; config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = xe->info.devid | (xe->info.revid << 16); - if (xe_device_get_root_tile(xe)->mem.vram.usable_size) + if (xe->mem.vram) config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM)) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index a7ff5975873f..e6871734ffa9 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -306,16 +306,11 @@ static struct xe_vram_region *page_to_vr(struct page *page) return container_of(page_pgmap(page), struct xe_vram_region, pagemap); } -static struct xe_tile *vr_to_tile(struct xe_vram_region *vr) -{ - return container_of(vr, struct xe_tile, mem.vram); -} - static u64 xe_vram_region_page_to_dpa(struct xe_vram_region *vr, struct page *page) { u64 dpa; - struct xe_tile *tile = vr_to_tile(vr); + struct xe_tile *tile = vr->tile; u64 pfn = page_to_pfn(page); u64 offset; @@ -370,7 +365,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, if (!vr && spage) { vr = page_to_vr(spage); - tile = vr_to_tile(vr); + tile = vr->tile; } XE_WARN_ON(spage && page_to_vr(spage) != vr); @@ -508,7 +503,7 @@ static u64 block_offset_to_pfn(struct xe_vram_region *vr, u64 offset) static struct drm_buddy *tile_to_buddy(struct xe_tile *tile) { - return &tile->mem.vram.ttm.mm; + return &tile->mem.vram->ttm.mm; } static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocation, @@ -522,7 +517,7 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati list_for_each_entry(block, blocks, link) { struct xe_vram_region *vr = block->private; - struct xe_tile *tile = vr_to_tile(vr); + struct xe_tile *tile = vr->tile; struct drm_buddy *buddy = tile_to_buddy(tile); u64 block_pfn = block_offset_to_pfn(vr, drm_buddy_block_offset(block)); int i; @@ -683,20 +678,15 @@ u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 start, u64 end, struct xe_vma *v } #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) -static struct xe_vram_region *tile_to_vr(struct xe_tile *tile) -{ - return &tile->mem.vram; -} - static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, unsigned long start, unsigned long end, struct mm_struct *mm, unsigned long timeslice_ms) { - struct xe_tile *tile = container_of(dpagemap, typeof(*tile), mem.vram.dpagemap); + struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); + struct xe_tile *tile = vr->tile; struct xe_device *xe = tile_to_xe(tile); struct device *dev = xe->drm.dev; - struct xe_vram_region *vr = tile_to_vr(tile); struct drm_buddy_block *block; struct list_head *blocks; struct xe_bo *bo; @@ -722,7 +712,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, &dpagemap_devmem_ops, - &tile->mem.vram.dpagemap, + &tile->mem.vram->dpagemap, end - start); blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 86e9811e60ba..858ce0183aaa 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -19,6 +19,7 @@ #include "xe_tile_sysfs.h" #include "xe_ttm_vram_mgr.h" #include "xe_wa.h" +#include "xe_vram.h" /** * DOC: Multi-tile Design @@ -95,6 +96,33 @@ static int xe_tile_alloc(struct xe_tile *tile) return 0; } +/** + * xe_tile_alloc_vram - Perform per-tile VRAM structs allocation + * @tile: Tile to perform allocations for + * + * Allocates VRAM per-tile data structures using DRM-managed allocations. + * Does not touch the hardware. + * + * Returns -ENOMEM if allocations fail, otherwise 0. + */ +int xe_tile_alloc_vram(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_vram_region *vram; + + if (!IS_DGFX(xe)) + return 0; + + vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); + if (!vram) + return -ENOMEM; + + vram->tile = tile; + tile->mem.vram = vram; + + return 0; +} + /** * xe_tile_init_early - Initialize the tile and primary GT * @tile: Tile to initialize @@ -132,8 +160,8 @@ static int tile_ttm_mgr_init(struct xe_tile *tile) struct xe_device *xe = tile_to_xe(tile); int err; - if (tile->mem.vram.usable_size) { - err = xe_ttm_vram_mgr_init(tile, &tile->mem.vram.ttm); + if (tile->mem.vram) { + err = xe_ttm_vram_mgr_init(tile, &tile->mem.vram->ttm); if (err) return err; xe->info.mem_region_mask |= BIT(tile->id) << 1; @@ -168,7 +196,7 @@ int xe_tile_init_noalloc(struct xe_tile *tile) xe_wa_apply_tile_workarounds(tile); if (xe->info.has_usm && IS_DGFX(xe)) - xe_devm_add(tile, &tile->mem.vram); + xe_devm_add(tile, tile->mem.vram); return xe_tile_sysfs_init(tile); } diff --git a/drivers/gpu/drm/xe/xe_tile.h b/drivers/gpu/drm/xe/xe_tile.h index cc33e8733983..440bc9e11c8b 100644 --- a/drivers/gpu/drm/xe/xe_tile.h +++ b/drivers/gpu/drm/xe/xe_tile.h @@ -14,12 +14,14 @@ int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id); int xe_tile_init_noalloc(struct xe_tile *tile); int xe_tile_init(struct xe_tile *tile); +int xe_tile_alloc_vram(struct xe_tile *tile); + void xe_tile_migrate_wait(struct xe_tile *tile); #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) { - return &tile->mem.vram.dpagemap; + return &tile->mem.vram->dpagemap; } #else static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c index d9c9d2547aad..9a9733447230 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -25,6 +25,7 @@ #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_vram_mgr.h" #include "xe_wa.h" +#include "xe_vram.h" struct xe_ttm_stolen_mgr { struct xe_ttm_vram_mgr base; @@ -82,15 +83,16 @@ static u32 get_wopcm_size(struct xe_device *xe) static s64 detect_bar2_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) { - struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_vram_region *tile_vram = xe_device_get_root_tile(xe)->mem.vram; + resource_size_t tile_io_start = xe_vram_region_io_start(tile_vram); struct xe_mmio *mmio = xe_root_tile_mmio(xe); struct pci_dev *pdev = to_pci_dev(xe->drm.dev); u64 stolen_size, wopcm_size; u64 tile_offset; u64 tile_size; - tile_offset = tile->mem.vram.io_start - xe->mem.vram.io_start; - tile_size = tile->mem.vram.actual_physical_size; + tile_offset = tile_io_start - xe_vram_region_io_start(xe->mem.vram); + tile_size = xe_vram_region_actual_physical_size(tile_vram); /* Use DSM base address instead for stolen memory */ mgr->stolen_base = (xe_mmio_read64_2x32(mmio, DSMBASE) & BDSM_MASK) - tile_offset; @@ -107,7 +109,7 @@ static s64 detect_bar2_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) /* Verify usage fits in the actual resource available */ if (mgr->stolen_base + stolen_size <= pci_resource_len(pdev, LMEM_BAR)) - mgr->io_base = tile->mem.vram.io_start + mgr->stolen_base; + mgr->io_base = tile_io_start + mgr->stolen_base; /* * There may be few KB of platform dependent reserved memory at the end diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 9e375a40aee9..3de2df47959b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -340,10 +340,11 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr) { struct xe_device *xe = tile_to_xe(tile); - struct xe_vram_region *vram = &tile->mem.vram; + struct xe_vram_region *vram = tile->mem.vram; return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id, - vram->usable_size, vram->io_size, + xe_vram_region_usable_size(vram), + xe_vram_region_io_size(vram), PAGE_SIZE); } @@ -392,7 +393,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, */ xe_res_first(res, offset, length, &cursor); for_each_sgtable_sg((*sgt), sg, i) { - phys_addr_t phys = cursor.start + tile->mem.vram.io_start; + phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram); size_t size = min_t(u64, cursor.size, SZ_2G); dma_addr_t addr; diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 3a4c84e9efc6..c93e9b96354b 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -3,6 +3,7 @@ * Copyright © 2021-2024 Intel Corporation */ +#include #include #include @@ -147,17 +148,17 @@ static int determine_lmem_bar_size(struct xe_device *xe) resize_vram_bar(xe); - xe->mem.vram.io_start = pci_resource_start(pdev, LMEM_BAR); - xe->mem.vram.io_size = pci_resource_len(pdev, LMEM_BAR); - if (!xe->mem.vram.io_size) + xe->mem.vram->io_start = pci_resource_start(pdev, LMEM_BAR); + xe->mem.vram->io_size = pci_resource_len(pdev, LMEM_BAR); + if (!xe->mem.vram->io_size) return -EIO; /* XXX: Need to change when xe link code is ready */ - xe->mem.vram.dpa_base = 0; + xe->mem.vram->dpa_base = 0; /* set up a map to the total memory area. */ - xe->mem.vram.mapping = devm_ioremap_wc(&pdev->dev, xe->mem.vram.io_start, - xe->mem.vram.io_size); + xe->mem.vram->mapping = devm_ioremap_wc(&pdev->dev, xe->mem.vram->io_start, + xe->mem.vram->io_size); return 0; } @@ -279,10 +280,10 @@ static void vram_fini(void *arg) struct xe_tile *tile; int id; - xe->mem.vram.mapping = NULL; + xe->mem.vram->mapping = NULL; for_each_tile(tile, xe, id) - tile->mem.vram.mapping = NULL; + tile->mem.vram->mapping = NULL; } /** @@ -318,10 +319,10 @@ int xe_vram_probe(struct xe_device *xe) if (err) return err; - drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &xe->mem.vram.io_start, - &xe->mem.vram.io_size); + drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &xe->mem.vram->io_start, + &xe->mem.vram->io_size); - io_size = xe->mem.vram.io_size; + io_size = xe->mem.vram->io_size; /* tile specific ranges */ for_each_tile(tile, xe, id) { @@ -329,45 +330,105 @@ int xe_vram_probe(struct xe_device *xe) if (err) return err; - tile->mem.vram.actual_physical_size = tile_size; - tile->mem.vram.io_start = xe->mem.vram.io_start + tile_offset; - tile->mem.vram.io_size = min_t(u64, vram_size, io_size); + tile->mem.vram->actual_physical_size = tile_size; + tile->mem.vram->io_start = xe->mem.vram->io_start + tile_offset; + tile->mem.vram->io_size = min_t(u64, vram_size, io_size); - if (!tile->mem.vram.io_size) { + if (!tile->mem.vram->io_size) { drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); return -ENODEV; } - tile->mem.vram.dpa_base = xe->mem.vram.dpa_base + tile_offset; - tile->mem.vram.usable_size = vram_size; - tile->mem.vram.mapping = xe->mem.vram.mapping + tile_offset; + tile->mem.vram->dpa_base = xe->mem.vram->dpa_base + tile_offset; + tile->mem.vram->usable_size = vram_size; + tile->mem.vram->mapping = xe->mem.vram->mapping + tile_offset; - if (tile->mem.vram.io_size < tile->mem.vram.usable_size) + if (tile->mem.vram->io_size < tile->mem.vram->usable_size) drm_info(&xe->drm, "Small BAR device\n"); - drm_info(&xe->drm, "VRAM[%u, %u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", id, - tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.usable_size, &tile->mem.vram.io_size); - drm_info(&xe->drm, "VRAM[%u, %u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", id, tile->id, - &tile->mem.vram.dpa_base, tile->mem.vram.dpa_base + (u64)tile->mem.vram.actual_physical_size, - &tile->mem.vram.io_start, tile->mem.vram.io_start + (u64)tile->mem.vram.io_size); + drm_info(&xe->drm, + "VRAM[%u, %u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", + id, tile->id, &tile->mem.vram->actual_physical_size, + &tile->mem.vram->usable_size, &tile->mem.vram->io_size); + drm_info(&xe->drm, "VRAM[%u, %u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", + id, tile->id, &tile->mem.vram->dpa_base, + tile->mem.vram->dpa_base + (u64)tile->mem.vram->actual_physical_size, + &tile->mem.vram->io_start, + tile->mem.vram->io_start + (u64)tile->mem.vram->io_size); /* calculate total size using tile size to get the correct HW sizing */ total_size += tile_size; available_size += vram_size; - if (total_size > xe->mem.vram.io_size) { + if (total_size > xe->mem.vram->io_size) { drm_info(&xe->drm, "VRAM: %pa is larger than resource %pa\n", - &total_size, &xe->mem.vram.io_size); + &total_size, &xe->mem.vram->io_size); } io_size -= min_t(u64, tile_size, io_size); } - xe->mem.vram.actual_physical_size = total_size; + xe->mem.vram->actual_physical_size = total_size; - drm_info(&xe->drm, "Total VRAM: %pa, %pa\n", &xe->mem.vram.io_start, - &xe->mem.vram.actual_physical_size); - drm_info(&xe->drm, "Available VRAM: %pa, %pa\n", &xe->mem.vram.io_start, + drm_info(&xe->drm, "Total VRAM: %pa, %pa\n", &xe->mem.vram->io_start, + &xe->mem.vram->actual_physical_size); + drm_info(&xe->drm, "Available VRAM: %pa, %pa\n", &xe->mem.vram->io_start, &available_size); return devm_add_action_or_reset(xe->drm.dev, vram_fini, xe); } + +/** + * xe_vram_region_io_start - Get the IO start of a VRAM region + * @vram: the VRAM region + * + * Return: the IO start of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram) +{ + return vram ? vram->io_start : 0; +} + +/** + * xe_vram_region_io_size - Get the IO size of a VRAM region + * @vram: the VRAM region + * + * Return: the IO size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram) +{ + return vram ? vram->io_size : 0; +} + +/** + * xe_vram_region_dpa_base - Get the DPA base of a VRAM region + * @vram: the VRAM region + * + * Return: the DPA base of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram) +{ + return vram ? vram->dpa_base : 0; +} + +/** + * xe_vram_region_usable_size - Get the usable size of a VRAM region + * @vram: the VRAM region + * + * Return: the usable size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram) +{ + return vram ? vram->usable_size : 0; +} + +/** + * xe_vram_region_actual_physical_size - Get the actual physical size of a VRAM region + * @vram: the VRAM region + * + * Return: the actual physical size of the VRAM region, or 0 if not valid + */ +resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram) +{ + return vram ? vram->actual_physical_size : 0; +} +EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size); diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h index e31cc04ec0db..d4bf1f9c2a72 100644 --- a/drivers/gpu/drm/xe/xe_vram.h +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -6,8 +6,17 @@ #ifndef _XE_VRAM_H_ #define _XE_VRAM_H_ +#include + struct xe_device; +struct xe_vram_region; int xe_vram_probe(struct xe_device *xe); +resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram); +resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram); + #endif From 7a20b4f558f4291161f71a5b7384262db9ccd6b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 14 Jul 2025 20:48:16 +0200 Subject: [PATCH 007/169] drm/xe: Move struct xe_vram_region to a dedicated header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's move the xe_vram_region structure to a new header dedicated to VRAM to improve modularity and avoid unnecessary dependencies when only VRAM-related structures are needed. v2: Fix build if CONFIG_DRM_XE_DEVMEM_MIRROR is enabled v3: Fix build if CONFIG_DRM_XE_DISPLAY is enabled v4: Move helper to get tile dpagemap to xe_svm.c Signed-off-by: Piotr Piórkowski Suggested-by: Jani Nikula Reviewed-by: Satyanarayana K V P # rev3 Acked-by: Matthew Brost Link: https://lore.kernel.org/r/20250714184818.89201-4-piotr.piorkowski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/display/xe_fb_pin.c | 1 + drivers/gpu/drm/xe/display/xe_plane_initial.c | 1 + drivers/gpu/drm/xe/xe_bo.c | 1 + drivers/gpu/drm/xe/xe_bo_types.h | 1 + drivers/gpu/drm/xe/xe_device.c | 1 + drivers/gpu/drm/xe/xe_device_types.h | 60 +-------------- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 1 + drivers/gpu/drm/xe/xe_svm.c | 8 +- drivers/gpu/drm/xe/xe_tile.c | 1 + drivers/gpu/drm/xe/xe_tile.h | 12 --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 1 + drivers/gpu/drm/xe/xe_vram.c | 1 + drivers/gpu/drm/xe/xe_vram_types.h | 74 +++++++++++++++++++ 13 files changed, 91 insertions(+), 72 deletions(-) create mode 100644 drivers/gpu/drm/xe/xe_vram_types.h diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index a1b203f40390..3f0362453e73 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -16,6 +16,7 @@ #include "xe_device.h" #include "xe_ggtt.h" #include "xe_pm.h" +#include "xe_vram_types.h" static void write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, diff --git a/drivers/gpu/drm/xe/display/xe_plane_initial.c b/drivers/gpu/drm/xe/display/xe_plane_initial.c index 30a341c96e76..54ff70ea2d25 100644 --- a/drivers/gpu/drm/xe/display/xe_plane_initial.c +++ b/drivers/gpu/drm/xe/display/xe_plane_initial.c @@ -21,6 +21,7 @@ #include "intel_frontbuffer.h" #include "intel_plane_initial.h" #include "xe_bo.h" +#include "xe_vram_types.h" #include "xe_wa.h" #include diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 18f27da47a36..00ce067d5fd3 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -36,6 +36,7 @@ #include "xe_trace_bo.h" #include "xe_ttm_stolen_mgr.h" #include "xe_vm.h" +#include "xe_vram_types.h" const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES] = { [XE_PL_SYSTEM] = "system", diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index ff560d82496f..57d34698139e 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index df6f1790d6a6..08f45777f797 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -64,6 +64,7 @@ #include "xe_ttm_sys_mgr.h" #include "xe_vm.h" #include "xe_vram.h" +#include "xe_vram_types.h" #include "xe_vsec.h" #include "xe_wait_user_fence.h" #include "xe_wa.h" diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 6299f4571c4c..eb6105523f23 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -10,7 +10,6 @@ #include #include -#include #include #include "xe_devcoredump_types.h" @@ -26,7 +25,6 @@ #include "xe_sriov_vf_types.h" #include "xe_step_types.h" #include "xe_survivability_mode_types.h" -#include "xe_ttm_vram_mgr_types.h" #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) #define TEST_VM_OPS_ERROR @@ -39,6 +37,7 @@ struct xe_ggtt; struct xe_i2c; struct xe_pat_ops; struct xe_pxp; +struct xe_vram_region; #define XE_BO_INVALID_OFFSET LONG_MAX @@ -71,63 +70,6 @@ struct xe_pxp; const struct xe_tile * : (const struct xe_device *)((tile__)->xe), \ struct xe_tile * : (tile__)->xe) -/** - * struct xe_vram_region - memory region structure - * This is used to describe a memory region in xe - * device, such as HBM memory or CXL extension memory. - */ -struct xe_vram_region { - /** @tile: Back pointer to tile */ - struct xe_tile *tile; - /** @io_start: IO start address of this VRAM instance */ - resource_size_t io_start; - /** - * @io_size: IO size of this VRAM instance - * - * This represents how much of this VRAM we can access - * via the CPU through the VRAM BAR. This can be smaller - * than @usable_size, in which case only part of VRAM is CPU - * accessible (typically the first 256M). This - * configuration is known as small-bar. - */ - resource_size_t io_size; - /** @dpa_base: This memory regions's DPA (device physical address) base */ - resource_size_t dpa_base; - /** - * @usable_size: usable size of VRAM - * - * Usable size of VRAM excluding reserved portions - * (e.g stolen mem) - */ - resource_size_t usable_size; - /** - * @actual_physical_size: Actual VRAM size - * - * Actual VRAM size including reserved portions - * (e.g stolen mem) - */ - resource_size_t actual_physical_size; - /** @mapping: pointer to VRAM mappable space */ - void __iomem *mapping; - /** @ttm: VRAM TTM manager */ - struct xe_ttm_vram_mgr ttm; -#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) - /** @pagemap: Used to remap device memory as ZONE_DEVICE */ - struct dev_pagemap pagemap; - /** - * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory - * pages of this tile. - */ - struct drm_pagemap dpagemap; - /** - * @hpa_base: base host physical address - * - * This is generated when remap device memory as ZONE_DEVICE - */ - resource_size_t hpa_base; -#endif -}; - /** * struct xe_mmio - register mmio structure * diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 86889df04011..f00cce81f1ff 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -33,6 +33,7 @@ #include "xe_migrate.h" #include "xe_sriov.h" #include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" #include "xe_wopcm.h" #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index e6871734ffa9..901f9a0268e6 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -17,6 +17,7 @@ #include "xe_ttm_vram_mgr.h" #include "xe_vm.h" #include "xe_vm_types.h" +#include "xe_vram_types.h" static bool xe_svm_range_in_vram(struct xe_svm_range *range) { @@ -989,6 +990,11 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile) +{ + return &tile->mem.vram->dpagemap; +} + /** * xe_svm_alloc_vram()- Allocate device memory pages for range, * migrating existing data. @@ -1006,7 +1012,7 @@ int xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, xe_assert(tile_to_xe(tile), range->base.flags.migrate_devmem); range_debug(range, "ALLOCATE VRAM"); - dpagemap = xe_tile_local_pagemap(tile); + dpagemap = tile_local_pagemap(tile); return drm_pagemap_populate_mm(dpagemap, xe_svm_range_start(range), xe_svm_range_end(range), range->base.gpusvm->mm, diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 858ce0183aaa..bd2ff91a7d1c 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -20,6 +20,7 @@ #include "xe_ttm_vram_mgr.h" #include "xe_wa.h" #include "xe_vram.h" +#include "xe_vram_types.h" /** * DOC: Multi-tile Design diff --git a/drivers/gpu/drm/xe/xe_tile.h b/drivers/gpu/drm/xe/xe_tile.h index 440bc9e11c8b..dceb6297aa01 100644 --- a/drivers/gpu/drm/xe/xe_tile.h +++ b/drivers/gpu/drm/xe/xe_tile.h @@ -18,18 +18,6 @@ int xe_tile_alloc_vram(struct xe_tile *tile); void xe_tile_migrate_wait(struct xe_tile *tile); -#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) -static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) -{ - return &tile->mem.vram->dpagemap; -} -#else -static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) -{ - return NULL; -} -#endif - static inline bool xe_tile_is_root(struct xe_tile *tile) { return tile->id == 0; diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 3de2df47959b..8f9b8a1d2c05 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -15,6 +15,7 @@ #include "xe_gt.h" #include "xe_res_cursor.h" #include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" static inline struct drm_buddy_block * xe_ttm_vram_mgr_first_block(struct list_head *list) diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index c93e9b96354b..366e5d8a85ca 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -21,6 +21,7 @@ #include "xe_module.h" #include "xe_sriov.h" #include "xe_vram.h" +#include "xe_vram_types.h" #define BAR_SIZE_SHIFT 20 diff --git a/drivers/gpu/drm/xe/xe_vram_types.h b/drivers/gpu/drm/xe/xe_vram_types.h new file mode 100644 index 000000000000..a01838236036 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vram_types.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_VRAM_TYPES_H_ +#define _XE_VRAM_TYPES_H_ + +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) +#include +#endif + +#include "xe_ttm_vram_mgr_types.h" + +struct xe_tile; + +/** + * struct xe_vram_region - memory region structure + * This is used to describe a memory region in xe + * device, such as HBM memory or CXL extension memory. + */ +struct xe_vram_region { + /** @tile: Back pointer to tile */ + struct xe_tile *tile; + /** @io_start: IO start address of this VRAM instance */ + resource_size_t io_start; + /** + * @io_size: IO size of this VRAM instance + * + * This represents how much of this VRAM we can access + * via the CPU through the VRAM BAR. This can be smaller + * than @usable_size, in which case only part of VRAM is CPU + * accessible (typically the first 256M). This + * configuration is known as small-bar. + */ + resource_size_t io_size; + /** @dpa_base: This memory regions's DPA (device physical address) base */ + resource_size_t dpa_base; + /** + * @usable_size: usable size of VRAM + * + * Usable size of VRAM excluding reserved portions + * (e.g stolen mem) + */ + resource_size_t usable_size; + /** + * @actual_physical_size: Actual VRAM size + * + * Actual VRAM size including reserved portions + * (e.g stolen mem) + */ + resource_size_t actual_physical_size; + /** @mapping: pointer to VRAM mappable space */ + void __iomem *mapping; + /** @ttm: VRAM TTM manager */ + struct xe_ttm_vram_mgr ttm; +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + /** @pagemap: Used to remap device memory as ZONE_DEVICE */ + struct dev_pagemap pagemap; + /** + * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory + * pages of this tile. + */ + struct drm_pagemap dpagemap; + /** + * @hpa_base: base host physical address + * + * This is generated when remap device memory as ZONE_DEVICE + */ + resource_size_t hpa_base; +#endif +}; + +#endif From d65ff1ec85355959e4ca452fba458687e4da583a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 14 Jul 2025 20:48:17 +0200 Subject: [PATCH 008/169] drm/xe: Split xe_migrate allocation from initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, xe_migrate_init handled both allocation and initialization, Lets moves allocation to xe_tile_alloc via a new xe_migrate_alloc function, and keep initialization in xe_migrate_init. This will allow the migration pointers to be passed to other structures before full initialization. Also replaces devm_kzalloc with drmm_kzalloc for better DRM-managed memory. Signed-off-by: Piotr Piórkowski Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250714184818.89201-5-piotr.piorkowski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_migrate.c | 25 ++++++++++++++++++------- drivers/gpu/drm/xe/xe_migrate.h | 1 + drivers/gpu/drm/xe/xe_tile.c | 4 ++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index d6186ac530cb..6a80ae6104dd 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -389,6 +389,23 @@ static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)); } +/** + * xe_migrate_alloc - Allocate a migrate struct for a given &xe_tile + * @tile: &xe_tile + * + * Allocates a &xe_migrate for a given tile. + * + * Return: &xe_migrate on success, or NULL when out of memory. + */ +struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile) +{ + struct xe_migrate *m = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*m), GFP_KERNEL); + + if (m) + m->tile = tile; + return m; +} + /** * xe_migrate_init() - Initialize a migrate context * @tile: Back-pointer to the tile we're initializing for. @@ -399,16 +416,10 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) { struct xe_device *xe = tile_to_xe(tile); struct xe_gt *primary_gt = tile->primary_gt; - struct xe_migrate *m; + struct xe_migrate *m = tile->migrate; struct xe_vm *vm; int err; - m = devm_kzalloc(xe->drm.dev, sizeof(*m), GFP_KERNEL); - if (!m) - return ERR_PTR(-ENOMEM); - - m->tile = tile; - /* Special layout, prepared below.. */ vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | XE_VM_FLAG_SET_TILE_ID(tile)); diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index fb9839c1bae0..74c60f55004a 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -93,6 +93,7 @@ struct xe_migrate_pt_update { u8 tile_id; }; +struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile); struct xe_migrate *xe_migrate_init(struct xe_tile *tile); struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index bd2ff91a7d1c..171de6a4ba95 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -94,6 +94,10 @@ static int xe_tile_alloc(struct xe_tile *tile) if (!tile->mem.ggtt) return -ENOMEM; + tile->migrate = xe_migrate_alloc(tile); + if (!tile->migrate) + return -ENOMEM; + return 0; } From 4b0a5f5ce7849aab7a67ba9f113ed75626f6de36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 14 Jul 2025 20:48:18 +0200 Subject: [PATCH 009/169] drm/xe: Unify the initialization of VRAM regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently in the drivers we have defined VRAM regions per device and per tile. Initialization of these regions is done in two completely different ways. To simplify the logic of the code and make it easier to add new regions in the future, let's unify the way we initialize VRAM regions. v2: - fix doc comments in struct xe_vram_region - remove unnecessary includes (Jani) v3: - move code from xe_vram_init_regions_managers to xe_tile_init_noalloc (Matthew) - replace ioremap_wc to devm_ioremap_wc for mapping VRAM BAR (Matthew) - Replace the tile id parameter with vram region in the xe_pf_begin function. v4: - remove tile back pointer from struct xe_vram_region - add new back pointers: xe and migarte to xe_vram_region Signed-off-by: Piotr Piórkowski Cc: Stuart Summers Cc: Matthew Auld Cc: Jani Nikula Reviewed-by: Matthew Auld # rev3 Acked-by: Matthew Brost Link: https://lore.kernel.org/r/20250714184818.89201-6-piotr.piorkowski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_bo.h | 4 +- drivers/gpu/drm/xe/xe_gt_pagefault.c | 13 ++- drivers/gpu/drm/xe/xe_query.c | 3 +- drivers/gpu/drm/xe/xe_svm.c | 43 ++++---- drivers/gpu/drm/xe/xe_tile.c | 37 +++---- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 16 ++- drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 3 +- drivers/gpu/drm/xe/xe_vram.c | 151 ++++++++++++++++----------- drivers/gpu/drm/xe/xe_vram.h | 2 + drivers/gpu/drm/xe/xe_vram_types.h | 17 ++- 10 files changed, 164 insertions(+), 125 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 02e8cde4c6b2..8cce413b5235 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -12,6 +12,7 @@ #include "xe_macros.h" #include "xe_vm_types.h" #include "xe_vm.h" +#include "xe_vram_types.h" #define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */ @@ -23,8 +24,9 @@ #define XE_BO_FLAG_VRAM_MASK (XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1) /* -- */ #define XE_BO_FLAG_STOLEN BIT(4) +#define XE_BO_FLAG_VRAM(vram) (XE_BO_FLAG_VRAM0 << ((vram)->id)) #define XE_BO_FLAG_VRAM_IF_DGFX(tile) (IS_DGFX(tile_to_xe(tile)) ? \ - XE_BO_FLAG_VRAM0 << (tile)->id : \ + XE_BO_FLAG_VRAM((tile)->mem.vram) : \ XE_BO_FLAG_SYSTEM) #define XE_BO_FLAG_GGTT BIT(5) #define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6) diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index 5a75d56d8558..ab43dec52776 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -23,6 +23,7 @@ #include "xe_svm.h" #include "xe_trace_bo.h" #include "xe_vm.h" +#include "xe_vram_types.h" struct pagefault { u64 page_addr; @@ -74,7 +75,7 @@ static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) } static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, - bool atomic, unsigned int id) + bool atomic, struct xe_vram_region *vram) { struct xe_bo *bo = xe_vma_bo(vma); struct xe_vm *vm = xe_vma_vm(vma); @@ -84,14 +85,16 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, if (err) return err; - if (atomic && IS_DGFX(vm->xe)) { + if (atomic && vram) { + xe_assert(vm->xe, IS_DGFX(vm->xe)); + if (xe_vma_is_userptr(vma)) { err = -EACCES; return err; } /* Migrate to VRAM, move should invalidate the VMA first */ - err = xe_bo_migrate(bo, XE_PL_VRAM0 + id); + err = xe_bo_migrate(bo, vram->placement); if (err) return err; } else if (bo) { @@ -138,7 +141,7 @@ retry_userptr: /* Lock VM and BOs dma-resv */ drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { - err = xe_pf_begin(&exec, vma, atomic, tile->id); + err = xe_pf_begin(&exec, vma, atomic, tile->mem.vram); drm_exec_retry_on_contention(&exec); if (xe_vm_validate_should_retry(&exec, err, &end)) err = -EAGAIN; @@ -573,7 +576,7 @@ static int handle_acc(struct xe_gt *gt, struct acc *acc) /* Lock VM and BOs dma-resv */ drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { - ret = xe_pf_begin(&exec, vma, true, tile->id); + ret = xe_pf_begin(&exec, vma, true, tile->mem.vram); drm_exec_retry_on_contention(&exec); if (ret) break; diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 66ea3403afbe..56ef3ee2806f 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -27,6 +27,7 @@ #include "xe_oa.h" #include "xe_pxp.h" #include "xe_ttm_vram_mgr.h" +#include "xe_vram_types.h" #include "xe_wa.h" static const u16 xe_to_user_engine_class[] = { @@ -410,7 +411,7 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query gt_list->gt_list[iter].near_mem_regions = 0x1; else gt_list->gt_list[iter].near_mem_regions = - BIT(gt_to_tile(gt)->id) << 1; + BIT(gt_to_tile(gt)->mem.vram->id) << 1; gt_list->gt_list[iter].far_mem_regions = xe->info.mem_region_mask ^ gt_list->gt_list[iter].near_mem_regions; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 901f9a0268e6..10c8a1bcb86e 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -311,12 +311,11 @@ static u64 xe_vram_region_page_to_dpa(struct xe_vram_region *vr, struct page *page) { u64 dpa; - struct xe_tile *tile = vr->tile; u64 pfn = page_to_pfn(page); u64 offset; - xe_tile_assert(tile, is_device_private_page(page)); - xe_tile_assert(tile, (pfn << PAGE_SHIFT) >= vr->hpa_base); + xe_assert(vr->xe, is_device_private_page(page)); + xe_assert(vr->xe, (pfn << PAGE_SHIFT) >= vr->hpa_base); offset = (pfn << PAGE_SHIFT) - vr->hpa_base; dpa = vr->dpa_base + offset; @@ -333,7 +332,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, unsigned long npages, const enum xe_svm_copy_dir dir) { struct xe_vram_region *vr = NULL; - struct xe_tile *tile; + struct xe_device *xe; struct dma_fence *fence = NULL; unsigned long i; #define XE_VRAM_ADDR_INVALID ~0x0ull @@ -366,7 +365,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, if (!vr && spage) { vr = page_to_vr(spage); - tile = vr->tile; + xe = vr->xe; } XE_WARN_ON(spage && page_to_vr(spage) != vr); @@ -398,18 +397,18 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, if (vram_addr != XE_VRAM_ADDR_INVALID) { if (sram) { - vm_dbg(&tile->xe->drm, + vm_dbg(&xe->drm, "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", vram_addr, (u64)dma_addr[pos], i - pos + incr); - __fence = xe_migrate_from_vram(tile->migrate, + __fence = xe_migrate_from_vram(vr->migrate, i - pos + incr, vram_addr, dma_addr + pos); } else { - vm_dbg(&tile->xe->drm, + vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", (u64)dma_addr[pos], vram_addr, i - pos + incr); - __fence = xe_migrate_to_vram(tile->migrate, + __fence = xe_migrate_to_vram(vr->migrate, i - pos + incr, dma_addr + pos, vram_addr); @@ -434,17 +433,17 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, /* Extra mismatched device page, copy it */ if (!match && last && vram_addr != XE_VRAM_ADDR_INVALID) { if (sram) { - vm_dbg(&tile->xe->drm, + vm_dbg(&xe->drm, "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", vram_addr, (u64)dma_addr[pos], 1); - __fence = xe_migrate_from_vram(tile->migrate, 1, + __fence = xe_migrate_from_vram(vr->migrate, 1, vram_addr, dma_addr + pos); } else { - vm_dbg(&tile->xe->drm, + vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", (u64)dma_addr[pos], vram_addr, 1); - __fence = xe_migrate_to_vram(tile->migrate, 1, + __fence = xe_migrate_to_vram(vr->migrate, 1, dma_addr + pos, vram_addr); } @@ -502,9 +501,9 @@ static u64 block_offset_to_pfn(struct xe_vram_region *vr, u64 offset) return PHYS_PFN(offset + vr->hpa_base); } -static struct drm_buddy *tile_to_buddy(struct xe_tile *tile) +static struct drm_buddy *vram_to_buddy(struct xe_vram_region *vram) { - return &tile->mem.vram->ttm.mm; + return &vram->ttm.mm; } static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocation, @@ -518,8 +517,7 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati list_for_each_entry(block, blocks, link) { struct xe_vram_region *vr = block->private; - struct xe_tile *tile = vr->tile; - struct drm_buddy *buddy = tile_to_buddy(tile); + struct drm_buddy *buddy = vram_to_buddy(vr); u64 block_pfn = block_offset_to_pfn(vr, drm_buddy_block_offset(block)); int i; @@ -685,8 +683,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, unsigned long timeslice_ms) { struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); - struct xe_tile *tile = vr->tile; - struct xe_device *xe = tile_to_xe(tile); + struct xe_device *xe = vr->xe; struct device *dev = xe->drm.dev; struct drm_buddy_block *block; struct list_head *blocks; @@ -700,9 +697,9 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, xe_pm_runtime_get(xe); retry: - bo = xe_bo_create_locked(tile_to_xe(tile), NULL, NULL, end - start, + bo = xe_bo_create_locked(vr->xe, NULL, NULL, end - start, ttm_bo_type_device, - XE_BO_FLAG_VRAM_IF_DGFX(tile) | + (IS_DGFX(xe) ? XE_BO_FLAG_VRAM(vr) : XE_BO_FLAG_SYSTEM) | XE_BO_FLAG_CPU_ADDR_MIRROR); if (IS_ERR(bo)) { err = PTR_ERR(bo); @@ -712,9 +709,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, } drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, - &dpagemap_devmem_ops, - &tile->mem.vram->dpagemap, - end - start); + &dpagemap_devmem_ops, dpagemap, end - start); blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; list_for_each_entry(block, blocks, link) diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 171de6a4ba95..0be0a5c57ef4 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -7,6 +7,7 @@ #include +#include "xe_bo.h" #include "xe_device.h" #include "xe_ggtt.h" #include "xe_gt.h" @@ -118,11 +119,9 @@ int xe_tile_alloc_vram(struct xe_tile *tile) if (!IS_DGFX(xe)) return 0; - vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); - if (!vram) - return -ENOMEM; - - vram->tile = tile; + vram = xe_vram_region_alloc(xe, tile->id, XE_PL_VRAM0 + tile->id); + if (IS_ERR(vram)) + return PTR_ERR(vram); tile->mem.vram = vram; return 0; @@ -160,21 +159,6 @@ int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id) } ALLOW_ERROR_INJECTION(xe_tile_init_early, ERRNO); /* See xe_pci_probe() */ -static int tile_ttm_mgr_init(struct xe_tile *tile) -{ - struct xe_device *xe = tile_to_xe(tile); - int err; - - if (tile->mem.vram) { - err = xe_ttm_vram_mgr_init(tile, &tile->mem.vram->ttm); - if (err) - return err; - xe->info.mem_region_mask |= BIT(tile->id) << 1; - } - - return 0; -} - /** * xe_tile_init_noalloc - Init tile up to the point where allocations can happen. * @tile: The tile to initialize. @@ -192,17 +176,20 @@ static int tile_ttm_mgr_init(struct xe_tile *tile) int xe_tile_init_noalloc(struct xe_tile *tile) { struct xe_device *xe = tile_to_xe(tile); - int err; - - err = tile_ttm_mgr_init(tile); - if (err) - return err; xe_wa_apply_tile_workarounds(tile); if (xe->info.has_usm && IS_DGFX(xe)) xe_devm_add(tile, tile->mem.vram); + if (IS_DGFX(xe) && !ttm_resource_manager_used(&tile->mem.vram->ttm.manager)) { + int err = xe_ttm_vram_mgr_init(xe, tile->mem.vram); + + if (err) + return err; + xe->info.mem_region_mask |= BIT(tile->mem.vram->id) << 1; + } + return xe_tile_sysfs_init(tile); } diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 8f9b8a1d2c05..9175b4a2214b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -338,12 +338,18 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr); } -int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr) +/** + * xe_ttm_vram_mgr_init - initialize TTM VRAM region + * @xe: pointer to Xe device + * @vram: pointer to xe_vram_region that contains the memory region attributes + * + * Initialize the Xe TTM for given @vram region using the given parameters. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram) { - struct xe_device *xe = tile_to_xe(tile); - struct xe_vram_region *vram = tile->mem.vram; - - return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id, + return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement, xe_vram_region_usable_size(vram), xe_vram_region_io_size(vram), PAGE_SIZE); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h index cc76050e376d..87b7fae5edba 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -11,11 +11,12 @@ enum dma_data_direction; struct xe_device; struct xe_tile; +struct xe_vram_region; int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, u32 mem_type, u64 size, u64 io_size, u64 default_page_size); -int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr); +int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram); int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, struct ttm_resource *res, u64 offset, u64 length, diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 366e5d8a85ca..b44ebf50fedb 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -20,6 +20,7 @@ #include "xe_mmio.h" #include "xe_module.h" #include "xe_sriov.h" +#include "xe_ttm_vram_mgr.h" #include "xe_vram.h" #include "xe_vram_types.h" @@ -138,7 +139,7 @@ static bool resource_is_valid(struct pci_dev *pdev, int bar) return true; } -static int determine_lmem_bar_size(struct xe_device *xe) +static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *lmem_bar) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); @@ -149,17 +150,16 @@ static int determine_lmem_bar_size(struct xe_device *xe) resize_vram_bar(xe); - xe->mem.vram->io_start = pci_resource_start(pdev, LMEM_BAR); - xe->mem.vram->io_size = pci_resource_len(pdev, LMEM_BAR); - if (!xe->mem.vram->io_size) + lmem_bar->io_start = pci_resource_start(pdev, LMEM_BAR); + lmem_bar->io_size = pci_resource_len(pdev, LMEM_BAR); + if (!lmem_bar->io_size) return -EIO; /* XXX: Need to change when xe link code is ready */ - xe->mem.vram->dpa_base = 0; + lmem_bar->dpa_base = 0; /* set up a map to the total memory area. */ - xe->mem.vram->mapping = devm_ioremap_wc(&pdev->dev, xe->mem.vram->io_start, - xe->mem.vram->io_size); + lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size); return 0; } @@ -287,6 +287,67 @@ static void vram_fini(void *arg) tile->mem.vram->mapping = NULL; } +struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement) +{ + struct xe_vram_region *vram; + struct drm_device *drm = &xe->drm; + + xe_assert(xe, id < xe->info.tile_count); + + vram = drmm_kzalloc(drm, sizeof(*vram), GFP_KERNEL); + if (!vram) + return NULL; + + vram->xe = xe; + vram->id = id; + vram->placement = placement; +#if defined(CONFIG_DRM_XE_PAGEMAP) + vram->migrate = xe->tiles[id].migrate; +#endif + return vram; +} + +static void print_vram_region_info(struct xe_device *xe, struct xe_vram_region *vram) +{ + struct drm_device *drm = &xe->drm; + + if (vram->io_size < vram->usable_size) + drm_info(drm, "Small BAR device\n"); + + drm_info(drm, + "VRAM[%u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", + vram->id, &vram->actual_physical_size, &vram->usable_size, &vram->io_size); + drm_info(drm, "VRAM[%u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", + vram->id, &vram->dpa_base, vram->dpa_base + (u64)vram->actual_physical_size, + &vram->io_start, vram->io_start + (u64)vram->io_size); +} + +static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, + struct xe_vram_region *lmem_bar, u64 offset, u64 usable_size, + u64 region_size, resource_size_t remain_io_size) +{ + /* Check if VRAM region is already initialized */ + if (vram->mapping) + return 0; + + vram->actual_physical_size = region_size; + vram->io_start = lmem_bar->io_start + offset; + vram->io_size = min_t(u64, usable_size, remain_io_size); + + if (!vram->io_size) { + drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); + return -ENODEV; + } + + vram->dpa_base = lmem_bar->dpa_base + offset; + vram->mapping = lmem_bar->mapping + offset; + vram->usable_size = usable_size; + + print_vram_region_info(xe, vram); + + return 0; +} + /** * xe_vram_probe() - Probe VRAM configuration * @xe: the &xe_device @@ -298,82 +359,52 @@ static void vram_fini(void *arg) int xe_vram_probe(struct xe_device *xe) { struct xe_tile *tile; - resource_size_t io_size; + struct xe_vram_region lmem_bar; + resource_size_t remain_io_size; u64 available_size = 0; u64 total_size = 0; - u64 tile_offset; - u64 tile_size; - u64 vram_size; int err; u8 id; if (!IS_DGFX(xe)) return 0; - /* Get the size of the root tile's vram for later accessibility comparison */ - tile = xe_device_get_root_tile(xe); - err = tile_vram_size(tile, &vram_size, &tile_size, &tile_offset); + err = determine_lmem_bar_size(xe, &lmem_bar); if (err) return err; + drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &lmem_bar.io_start, &lmem_bar.io_size); - err = determine_lmem_bar_size(xe); - if (err) - return err; + remain_io_size = lmem_bar.io_size; - drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &xe->mem.vram->io_start, - &xe->mem.vram->io_size); - - io_size = xe->mem.vram->io_size; - - /* tile specific ranges */ for_each_tile(tile, xe, id) { - err = tile_vram_size(tile, &vram_size, &tile_size, &tile_offset); + u64 region_size; + u64 usable_size; + u64 tile_offset; + + err = tile_vram_size(tile, &usable_size, ®ion_size, &tile_offset); if (err) return err; - tile->mem.vram->actual_physical_size = tile_size; - tile->mem.vram->io_start = xe->mem.vram->io_start + tile_offset; - tile->mem.vram->io_size = min_t(u64, vram_size, io_size); + total_size += region_size; + available_size += usable_size; - if (!tile->mem.vram->io_size) { - drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); - return -ENODEV; - } + err = vram_region_init(xe, tile->mem.vram, &lmem_bar, tile_offset, usable_size, + region_size, remain_io_size); + if (err) + return err; - tile->mem.vram->dpa_base = xe->mem.vram->dpa_base + tile_offset; - tile->mem.vram->usable_size = vram_size; - tile->mem.vram->mapping = xe->mem.vram->mapping + tile_offset; - - if (tile->mem.vram->io_size < tile->mem.vram->usable_size) - drm_info(&xe->drm, "Small BAR device\n"); - drm_info(&xe->drm, - "VRAM[%u, %u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", - id, tile->id, &tile->mem.vram->actual_physical_size, - &tile->mem.vram->usable_size, &tile->mem.vram->io_size); - drm_info(&xe->drm, "VRAM[%u, %u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", - id, tile->id, &tile->mem.vram->dpa_base, - tile->mem.vram->dpa_base + (u64)tile->mem.vram->actual_physical_size, - &tile->mem.vram->io_start, - tile->mem.vram->io_start + (u64)tile->mem.vram->io_size); - - /* calculate total size using tile size to get the correct HW sizing */ - total_size += tile_size; - available_size += vram_size; - - if (total_size > xe->mem.vram->io_size) { + if (total_size > lmem_bar.io_size) { drm_info(&xe->drm, "VRAM: %pa is larger than resource %pa\n", - &total_size, &xe->mem.vram->io_size); + &total_size, &lmem_bar.io_size); } - io_size -= min_t(u64, tile_size, io_size); + remain_io_size -= min_t(u64, tile->mem.vram->actual_physical_size, remain_io_size); } - xe->mem.vram->actual_physical_size = total_size; - - drm_info(&xe->drm, "Total VRAM: %pa, %pa\n", &xe->mem.vram->io_start, - &xe->mem.vram->actual_physical_size); - drm_info(&xe->drm, "Available VRAM: %pa, %pa\n", &xe->mem.vram->io_start, - &available_size); + err = vram_region_init(xe, xe->mem.vram, &lmem_bar, 0, available_size, total_size, + lmem_bar.io_size); + if (err) + return err; return devm_add_action_or_reset(xe->drm.dev, vram_fini, xe); } diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h index d4bf1f9c2a72..72860f714fc6 100644 --- a/drivers/gpu/drm/xe/xe_vram.h +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -13,6 +13,8 @@ struct xe_vram_region; int xe_vram_probe(struct xe_device *xe); +struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement); + resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram); resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram); resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram); diff --git a/drivers/gpu/drm/xe/xe_vram_types.h b/drivers/gpu/drm/xe/xe_vram_types.h index a01838236036..83772dcbf1af 100644 --- a/drivers/gpu/drm/xe/xe_vram_types.h +++ b/drivers/gpu/drm/xe/xe_vram_types.h @@ -12,7 +12,8 @@ #include "xe_ttm_vram_mgr_types.h" -struct xe_tile; +struct xe_device; +struct xe_migrate; /** * struct xe_vram_region - memory region structure @@ -20,8 +21,14 @@ struct xe_tile; * device, such as HBM memory or CXL extension memory. */ struct xe_vram_region { - /** @tile: Back pointer to tile */ - struct xe_tile *tile; + /** @xe: Back pointer to xe device */ + struct xe_device *xe; + /** + * @id: VRAM region instance id + * + * The value should be unique for VRAM region. + */ + u8 id; /** @io_start: IO start address of this VRAM instance */ resource_size_t io_start; /** @@ -54,7 +61,11 @@ struct xe_vram_region { void __iomem *mapping; /** @ttm: VRAM TTM manager */ struct xe_ttm_vram_mgr ttm; + /** @placement: TTM placement dedicated for this region */ + u32 placement; #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + /** @migrate: Back pointer to migrate */ + struct xe_migrate *migrate; /** @pagemap: Used to remap device memory as ZONE_DEVICE */ struct dev_pagemap pagemap; /** From 487579fd8524c58dc237904c1b1d9e7c89e53378 Mon Sep 17 00:00:00 2001 From: Soham Purkait Date: Wed, 16 Jul 2025 15:44:12 +0530 Subject: [PATCH 010/169] drm/xe/xe_debugfs: Exposure of G-State and pcie link state residency counters through debugfs Add debug nodes, "dgfx_pkg_residencies" for G-states (G2, G6, G8, G10, ModS) and "dgfx_pcie_link_residencies" for PCIe link states(L0, L1, L1.2) residency counters. v1: - Expose all G-State residency counter values under dgfx_pkg_residencies. (Anshuman) - Include runtime_get/put. (Riana) v2: - Move offset macros to drm/xe/regs/xe_pmt. (Riana) v3: - Include debugfs node "dgfx_pcie_link_residencies" for pcie link residency counter values. (Anshuman) v4: - Include check for BMG and add helper function for repetitive code. (Riana) - Add for loop and local struct to avoid repetition. (Riana) - Use "drm_debugfs_create_files" to create debugfs. (Karthik) v5: - Reorder commits to reflect the correct dependency hierarchy. (Jonathan) - Simplification of commit message and rectified register offset.(Karthik) - Error handling and return before printing. (Riana) v6: - Remove check for DGFX as BMG is discrete. (Karthik) - Rearrange residency offsets in ascending order. (Riana) v7: - Squash the macros into the patch they are used in. (Lucas) Signed-off-by: Soham Purkait Reviewed-by: Jonathan Cavitt Reviewed-by: Karthik Poosa Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250716101412.3062780-2-soham.purkait@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/regs/xe_pmt.h | 10 ++++ drivers/gpu/drm/xe/xe_debugfs.c | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_pmt.h b/drivers/gpu/drm/xe/regs/xe_pmt.h index 2995d72c3f78..264e9baf949c 100644 --- a/drivers/gpu/drm/xe/regs/xe_pmt.h +++ b/drivers/gpu/drm/xe/regs/xe_pmt.h @@ -21,4 +21,14 @@ #define SG_REMAP_INDEX1 XE_REG(SOC_BASE + 0x08) #define SG_REMAP_BITS REG_GENMASK(31, 24) +#define BMG_MODS_RESIDENCY_OFFSET (0x4D0) +#define BMG_G2_RESIDENCY_OFFSET (0x530) +#define BMG_G6_RESIDENCY_OFFSET (0x538) +#define BMG_G8_RESIDENCY_OFFSET (0x540) +#define BMG_G10_RESIDENCY_OFFSET (0x548) + +#define BMG_PCIE_LINK_L0_RESIDENCY_OFFSET (0x570) +#define BMG_PCIE_LINK_L1_RESIDENCY_OFFSET (0x578) +#define BMG_PCIE_LINK_L1_2_RESIDENCY_OFFSET (0x580) + #endif diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 129186f4193b..0b4a532f7c45 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -11,18 +11,21 @@ #include +#include "regs/xe_pmt.h" #include "xe_bo.h" #include "xe_device.h" #include "xe_force_wake.h" #include "xe_gt_debugfs.h" #include "xe_gt_printk.h" #include "xe_guc_ads.h" +#include "xe_mmio.h" #include "xe_pm.h" #include "xe_pxp_debugfs.h" #include "xe_sriov.h" #include "xe_sriov_pf.h" #include "xe_step.h" #include "xe_wa.h" +#include "xe_vsec.h" #ifdef CONFIG_DRM_XE_DEBUG #include "xe_bo_evict.h" @@ -32,6 +35,23 @@ DECLARE_FAULT_ATTR(gt_reset_failure); +static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio, + u32 offset, char *name, struct drm_printer *p) +{ + u64 residency = 0; + int ret; + + ret = xe_pmt_telem_read(to_pci_dev(xe->drm.dev), + xe_mmio_read32(mmio, PUNIT_TELEMETRY_GUID), + &residency, offset, sizeof(residency)); + if (ret != sizeof(residency)) { + drm_warn(&xe->drm, "%s counter failed to read, ret %d\n", name, ret); + return; + } + + drm_printf(p, "%s : %llu\n", name, residency); +} + static struct xe_device *node_to_xe(struct drm_info_node *node) { return to_xe_device(node->minor->dev); @@ -102,12 +122,72 @@ static int workaround_info(struct seq_file *m, void *data) return 0; } +static int dgfx_pkg_residencies_show(struct seq_file *m, void *data) +{ + struct xe_device *xe; + struct xe_mmio *mmio; + struct drm_printer p; + + xe = node_to_xe(m->private); + p = drm_seq_file_printer(m); + xe_pm_runtime_get(xe); + mmio = xe_root_tile_mmio(xe); + struct { + u32 offset; + char *name; + } residencies[] = { + {BMG_G2_RESIDENCY_OFFSET, "Package G2"}, + {BMG_G6_RESIDENCY_OFFSET, "Package G6"}, + {BMG_G8_RESIDENCY_OFFSET, "Package G8"}, + {BMG_G10_RESIDENCY_OFFSET, "Package G10"}, + {BMG_MODS_RESIDENCY_OFFSET, "Package ModS"} + }; + + for (int i = 0; i < ARRAY_SIZE(residencies); i++) + read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); + + xe_pm_runtime_put(xe); + return 0; +} + +static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data) +{ + struct xe_device *xe; + struct xe_mmio *mmio; + struct drm_printer p; + + xe = node_to_xe(m->private); + p = drm_seq_file_printer(m); + xe_pm_runtime_get(xe); + mmio = xe_root_tile_mmio(xe); + + struct { + u32 offset; + char *name; + } residencies[] = { + {BMG_PCIE_LINK_L0_RESIDENCY_OFFSET, "PCIE LINK L0 RESIDENCY"}, + {BMG_PCIE_LINK_L1_RESIDENCY_OFFSET, "PCIE LINK L1 RESIDENCY"}, + {BMG_PCIE_LINK_L1_2_RESIDENCY_OFFSET, "PCIE LINK L1.2 RESIDENCY"} + }; + + for (int i = 0; i < ARRAY_SIZE(residencies); i++) + read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); + + xe_pm_runtime_put(xe); + return 0; +} + static const struct drm_info_list debugfs_list[] = { {"info", info, 0}, { .name = "sriov_info", .show = sriov_info, }, { .name = "workarounds", .show = workaround_info, }, }; +static const struct drm_info_list debugfs_residencies[] = { + { .name = "dgfx_pkg_residencies", .show = dgfx_pkg_residencies_show, }, + { .name = "dgfx_pcie_link_residencies", .show = dgfx_pcie_link_residencies_show, }, +}; + static int forcewake_open(struct inode *inode, struct file *file) { struct xe_device *xe = inode->i_private; @@ -280,6 +360,11 @@ void xe_debugfs_register(struct xe_device *xe) ARRAY_SIZE(debugfs_list), root, minor); + if (xe->info.platform == XE_BATTLEMAGE) + drm_debugfs_create_files(debugfs_residencies, + ARRAY_SIZE(debugfs_residencies), + root, minor); + debugfs_create_file("forcewake_all", 0400, root, xe, &forcewake_all_fops); From 4b5514f786dd790ec0d3e2fe66cea7fd2f36bdf3 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Fri, 18 Jul 2025 13:29:47 -0700 Subject: [PATCH 011/169] drm/xe: Remove unnecessary EU stall debug message The EU stall debug message may cause CI to complain on unsupported platforms. Remove it. Cc: Ashutosh Dixit Signed-off-by: Harish Chegondi Reviewed-by: Ashutosh Dixit Signed-off-by: Ashutosh Dixit Link: https://lore.kernel.org/r/dfb6a080b3442d481c567489aabe47e72f3e784c.1752870172.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_query.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 56ef3ee2806f..44d44bbc71dc 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -749,10 +749,8 @@ static int query_eu_stall(struct xe_device *xe, u32 num_rates; int ret; - if (!xe_eu_stall_supported_on_platform(xe)) { - drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + if (!xe_eu_stall_supported_on_platform(xe)) return -ENODEV; - } array_size = xe_eu_stall_get_sampling_rates(&num_rates, &rates); size = sizeof(struct drm_xe_query_eu_stall) + array_size; From 6c9e64e83b22405622d1f47417cdb0d20d49ca35 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 18 Jul 2025 16:23:05 -0500 Subject: [PATCH 012/169] drm/xe: Fix an IS_ERR() vs NULL bug in xe_tile_alloc_vram() The xe_vram_region_alloc() function returns NULL on error. It never returns error pointers. Update the error checking to match. Fixes: 4b0a5f5ce784 ("drm/xe: Unify the initialization of VRAM regions") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/5449065e-9758-4711-b706-78771c0753c4@sabinyo.mountain Reviewed-by: Rodrigo Vivi Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_tile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 0be0a5c57ef4..d49ba3401963 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -120,8 +120,8 @@ int xe_tile_alloc_vram(struct xe_tile *tile) return 0; vram = xe_vram_region_alloc(xe, tile->id, XE_PL_VRAM0 + tile->id); - if (IS_ERR(vram)) - return PTR_ERR(vram); + if (!vram) + return -ENOMEM; tile->mem.vram = vram; return 0; From 176f44a5ec0b074aaf44852db77d0c183c36696d Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Mon, 21 Jul 2025 17:45:20 -0400 Subject: [PATCH 013/169] drm/xe/uc: Fix missing unwind goto Fix missing unwind goto on error handling. Fixes: b2c4ac219fa4 ("drm/xe/uc: Disable GuC communication on hardware initialization error") Signed-off-by: Zhanjun Dong Reviewed-by: Matthew Brost Signed-off-by: John Harrison Link: https://lore.kernel.org/r/20250721214520.954014-1-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_uc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c index 3e0c3af235f2..465bda355443 100644 --- a/drivers/gpu/drm/xe/xe_uc.c +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -164,7 +164,7 @@ static int vf_uc_load_hw(struct xe_uc *uc) err = xe_guc_opt_in_features_enable(&uc->guc); if (err) - return err; + goto err_out; err = xe_gt_record_default_lrcs(uc_to_gt(uc)); if (err) From 9a220e065914b67b55d3d0ab91c3e215742fdd73 Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Thu, 17 Jul 2025 17:54:20 +0200 Subject: [PATCH 014/169] drm/xe/vf: Don't register I2C devices if VF VF drivers can't access I2C devices, so skip their registration when running as VF. Signed-off-by: Lukasz Laguna Fixes: f0e53aadd702 ("drm/xe: Support for I2C attached MCUs") Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250717155420.25298-1-lukasz.laguna@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_i2c.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index 1f19718db559..bc7dc2099470 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -283,6 +283,9 @@ int xe_i2c_probe(struct xe_device *xe) if (xe->info.platform != XE_BATTLEMAGE) return 0; + if (IS_SRIOV_VF(xe)) + return 0; + xe_i2c_read_endpoint(xe_root_tile_mmio(xe), &ep); if (ep.cookie != XE_I2C_EP_COOKIE_DEVICE) return 0; From f3009272ff2ec0498426001e4dd2a12c50234e08 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 22 Jul 2025 17:35:04 +0530 Subject: [PATCH 015/169] drm/xe/vf: Create contexts for CCS read write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create two LRCs to handle CCS meta data read / write from CCS pool in the VM. Read context is used to hold GPU instructions to be executed at save time and write context is used to hold GPU instructions to be executed at the restore time. Allocate batch buffer pool using suballocator for both read and write contexts. Migration framework is reused to create LRCAs for read and write. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Michał Winiarski Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250722120506.6483-2-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_device.c | 4 + drivers/gpu/drm/xe/xe_device_types.h | 4 + drivers/gpu/drm/xe/xe_gt_debugfs.c | 39 ++++ drivers/gpu/drm/xe/xe_sriov.c | 19 ++ drivers/gpu/drm/xe/xe_sriov.h | 1 + drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 208 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h | 13 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h | 45 +++++ drivers/gpu/drm/xe/xe_sriov_vf_types.h | 6 + 10 files changed, 340 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs.c create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs.h create mode 100644 drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 74231ca3fb9a..b046e1b8d726 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -149,6 +149,7 @@ xe-y += \ xe_memirq.o \ xe_sriov.o \ xe_sriov_vf.o \ + xe_sriov_vf_ccs.o \ xe_tile_sriov_vf.o xe-$(CONFIG_PCI_IOV) += \ diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 08f45777f797..d04a0ae018e6 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -940,6 +940,10 @@ int xe_device_probe(struct xe_device *xe) xe_vsec_init(xe); + err = xe_sriov_late_init(xe); + if (err) + goto err_unregister_display; + return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); err_unregister_display: diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index eb6105523f23..38c8329b4d2c 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -23,6 +23,7 @@ #include "xe_sriov_pf_types.h" #include "xe_sriov_types.h" #include "xe_sriov_vf_types.h" +#include "xe_sriov_vf_ccs_types.h" #include "xe_step_types.h" #include "xe_survivability_mode_types.h" @@ -182,6 +183,9 @@ struct xe_tile { struct { /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */ struct xe_ggtt_node *ggtt_balloon[2]; + + /** @sriov.vf.ccs: CCS read and write contexts for VF. */ + struct xe_tile_vf_ccs ccs[XE_SRIOV_VF_CCS_CTX_COUNT]; } vf; } sriov; diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 061a81f82e76..8d4b66ca38b7 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -134,6 +134,33 @@ static int sa_info(struct xe_gt *gt, struct drm_printer *p) return 0; } +static int sa_info_vf_ccs(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_sa_manager *bb_pool; + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + + if (!IS_VF_CCS_READY(gt_to_xe(gt))) + return 0; + + xe_pm_runtime_get(gt_to_xe(gt)); + + for_each_ccs_rw_ctx(ctx_id) { + bb_pool = tile->sriov.vf.ccs[ctx_id].mem.ccs_bb_pool; + if (!bb_pool) + break; + + drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read"); + drm_printf(p, "-------------------------\n"); + drm_suballoc_dump_debug_info(&bb_pool->base, p, bb_pool->gpu_addr); + drm_puts(p, "\n"); + } + + xe_pm_runtime_put(gt_to_xe(gt)); + + return 0; +} + static int topology(struct xe_gt *gt, struct drm_printer *p) { xe_pm_runtime_get(gt_to_xe(gt)); @@ -303,6 +330,13 @@ static const struct drm_info_list vf_safe_debugfs_list[] = { {"hwconfig", .show = xe_gt_debugfs_simple_show, .data = hwconfig}, }; +/* + * only for GT debugfs files which are valid on VF. Not valid on PF. + */ +static const struct drm_info_list vf_only_debugfs_list[] = { + {"sa_info_vf_ccs", .show = xe_gt_debugfs_simple_show, .data = sa_info_vf_ccs}, +}; + /* everything else should be added here */ static const struct drm_info_list pf_only_debugfs_list[] = { {"hw_engines", .show = xe_gt_debugfs_simple_show, .data = hw_engines}, @@ -424,6 +458,11 @@ void xe_gt_debugfs_register(struct xe_gt *gt) drm_debugfs_create_files(pf_only_debugfs_list, ARRAY_SIZE(pf_only_debugfs_list), root, minor); + else + drm_debugfs_create_files(vf_only_debugfs_list, + ARRAY_SIZE(vf_only_debugfs_list), + root, minor); + xe_uc_debugfs_register(>->uc, root); diff --git a/drivers/gpu/drm/xe/xe_sriov.c b/drivers/gpu/drm/xe/xe_sriov.c index a0eab44c0e76..87911fb4eea7 100644 --- a/drivers/gpu/drm/xe/xe_sriov.c +++ b/drivers/gpu/drm/xe/xe_sriov.c @@ -15,6 +15,7 @@ #include "xe_sriov.h" #include "xe_sriov_pf.h" #include "xe_sriov_vf.h" +#include "xe_sriov_vf_ccs.h" /** * xe_sriov_mode_to_string - Convert enum value to string. @@ -157,3 +158,21 @@ const char *xe_sriov_function_name(unsigned int n, char *buf, size_t size) strscpy(buf, "PF", size); return buf; } + +/** + * xe_sriov_late_init() - SR-IOV late initialization functions. + * @xe: the &xe_device to initialize + * + * On VF this function will initialize code for CCS migration. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_late_init(struct xe_device *xe) +{ + int err = 0; + + if (IS_VF_CCS_INIT_NEEDED(xe)) + err = xe_sriov_vf_ccs_init(xe); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_sriov.h b/drivers/gpu/drm/xe/xe_sriov.h index 688fbabf08f1..0e0c1abf2d14 100644 --- a/drivers/gpu/drm/xe/xe_sriov.h +++ b/drivers/gpu/drm/xe/xe_sriov.h @@ -18,6 +18,7 @@ const char *xe_sriov_function_name(unsigned int n, char *buf, size_t len); void xe_sriov_probe_early(struct xe_device *xe); void xe_sriov_print_info(struct xe_device *xe, struct drm_printer *p); int xe_sriov_init(struct xe_device *xe); +int xe_sriov_late_init(struct xe_device *xe); static inline enum xe_sriov_mode xe_device_sriov_mode(const struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c new file mode 100644 index 000000000000..697eea46659a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "instructions/xe_mi_commands.h" +#include "instructions/xe_gpu_commands.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_migrate.h" +#include "xe_sa.h" +#include "xe_sriov_printk.h" +#include "xe_sriov_vf_ccs.h" +#include "xe_sriov_vf_ccs_types.h" + +/** + * DOC: VF save/restore of compression Meta Data + * + * VF KMD registers two special contexts/LRCAs. + * + * Save Context/LRCA: contain necessary cmds+page table to trigger Meta data / + * compression control surface (Aka CCS) save in regular System memory in VM. + * + * Restore Context/LRCA: contain necessary cmds+page table to trigger Meta data / + * compression control surface (Aka CCS) Restore from regular System memory in + * VM to corresponding CCS pool. + * + * Below diagram explain steps needed for VF save/Restore of compression Meta Data:: + * + * CCS Save CCS Restore VF KMD Guc BCS + * LRCA LRCA + * | | | | | + * | | | | | + * | Create Save LRCA | | | + * [ ]<----------------------------- [ ] | | + * | | | | | + * | | | | | + * | | | Register save LRCA | | + * | | | with Guc | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | Create restore LRCA | | | + * | [ ]<------------------[ ] | | + * | | | | | + * | | | Register restore LRCA | | + * | | | with Guc | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | | | | + * | | [ ]------------------------- | | + * | | [ ] Allocate main memory. | | | + * | | [ ] Allocate CCS memory. | | | + * | | [ ] Update Main memory & | | | + * [ ]<------------------------------[ ] CCS pages PPGTT + BB | | | + * | [ ]<------------------[ ] cmds to save & restore.| | | + * | | [ ]<------------------------ | | + * | | | | | + * | | | | | + * | | | | | + * : : : : : + * ---------------------------- VF Paused ------------------------------------- + * | | | | | + * | | | | | + * | | | |Schedule | + * | | | |CCS Save | + * | | | | LRCA | + * | | | [ ]------>[ ] + * | | | | | + * | | | | | + * | | | |CCS save | + * | | | |completed| + * | | | [ ]<------[ ] + * | | | | | + * : : : : : + * ---------------------------- VM Migrated ----------------------------------- + * | | | | | + * | | | | | + * : : : : : + * ---------------------------- VF Resumed ------------------------------------ + * | | | | | + * | | | | | + * | | [ ]-------------- | | + * | | [ ] Fix up GGTT | | | + * | | [ ]<------------- | | + * | | | | | + * | | | | | + * | | | Notify VF_RESFIX_DONE | | + * | | [ ]--------------------------->[ ] | + * | | | | | + * | | | |Schedule | + * | | | |CCS | + * | | | |Restore | + * | | | |LRCA | + * | | | [ ]------>[ ] + * | | | | | + * | | | | | + * | | | |CCS | + * | | | |restore | + * | | | |completed| + * | | | [ ]<------[ ] + * | | | | | + * | | | | | + * | | | VF_RESFIX_DONE complete | | + * | | | notification | | + * | | [ ]<---------------------------[ ] | + * | | | | | + * | | | | | + * : : : : : + * ------------------------- Continue VM restore ------------------------------ + */ + +static u64 get_ccs_bb_pool_size(struct xe_device *xe) +{ + u64 sys_mem_size, ccs_mem_size, ptes, bb_pool_size; + struct sysinfo si; + + si_meminfo(&si); + sys_mem_size = si.totalram * si.mem_unit; + ccs_mem_size = div64_u64(sys_mem_size, NUM_BYTES_PER_CCS_BYTE(xe)); + ptes = DIV_ROUND_UP_ULL(sys_mem_size + ccs_mem_size, XE_PAGE_SIZE); + + /** + * We need below BB size to hold PTE mappings and some DWs for copy + * command. In reality, we need space for many copy commands. So, let + * us allocate double the calculated size which is enough to holds GPU + * instructions for the whole region. + */ + bb_pool_size = ptes * sizeof(u32); + + return round_up(bb_pool_size * 2, SZ_1M); +} + +static int alloc_bb_pool(struct xe_tile *tile, struct xe_tile_vf_ccs *ctx) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_sa_manager *sa_manager; + u64 bb_pool_size; + int offset, err; + + bb_pool_size = get_ccs_bb_pool_size(xe); + xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", + ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); + + sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16); + + if (IS_ERR(sa_manager)) { + xe_sriov_err(xe, "Suballocator init failed with error: %pe\n", + sa_manager); + err = PTR_ERR(sa_manager); + return err; + } + + offset = 0; + xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP, + bb_pool_size); + + offset = bb_pool_size - sizeof(u32); + xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END); + + ctx->mem.ccs_bb_pool = sa_manager; + + return 0; +} + +/** + * xe_sriov_vf_ccs_init - Setup LRCA for save & restore. + * @xe: the &xe_device to start recovery on + * + * This function shall be called only by VF. It initializes + * LRCA and suballocator needed for CCS save & restore. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_sriov_vf_ccs_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_migrate *migrate; + struct xe_tile_vf_ccs *ctx; + int err; + + xe_assert(xe, IS_SRIOV_VF(xe)); + xe_assert(xe, !IS_DGFX(xe)); + xe_assert(xe, xe_device_has_flat_ccs(xe)); + + for_each_ccs_rw_ctx(ctx_id) { + ctx = &tile->sriov.vf.ccs[ctx_id]; + ctx->ctx_id = ctx_id; + + migrate = xe_migrate_init(tile); + if (IS_ERR(migrate)) { + err = PTR_ERR(migrate); + goto err_ret; + } + ctx->migrate = migrate; + + err = alloc_bb_pool(tile, ctx); + if (err) + goto err_ret; + } + + xe->sriov.vf.ccs.initialized = 1; + + return 0; + +err_ret: + return err; +} diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h new file mode 100644 index 000000000000..5df9ba028d14 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_CCS_H_ +#define _XE_SRIOV_VF_CCS_H_ + +struct xe_device; + +int xe_sriov_vf_ccs_init(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h new file mode 100644 index 000000000000..6dc279d206ec --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VF_CCS_TYPES_H_ +#define _XE_SRIOV_VF_CCS_TYPES_H_ + +#define for_each_ccs_rw_ctx(id__) \ + for ((id__) = 0; (id__) < XE_SRIOV_VF_CCS_CTX_COUNT; (id__)++) + +#define IS_VF_CCS_READY(xe) ({ \ + struct xe_device *___xe = (xe); \ + xe_assert(___xe, IS_SRIOV_VF(___xe)); \ + ___xe->sriov.vf.ccs.initialized; \ + }) + +#define IS_VF_CCS_INIT_NEEDED(xe) ({\ + struct xe_device *___xe = (xe); \ + IS_SRIOV_VF(___xe) && !IS_DGFX(___xe) && \ + xe_device_has_flat_ccs(___xe) && GRAPHICS_VER(___xe) >= 20; \ + }) + +enum xe_sriov_vf_ccs_rw_ctxs { + XE_SRIOV_VF_CCS_READ_CTX, + XE_SRIOV_VF_CCS_WRITE_CTX, + XE_SRIOV_VF_CCS_CTX_COUNT +}; + +struct xe_migrate; +struct xe_sa_manager; + +struct xe_tile_vf_ccs { + /** @id: Id to which context it belongs to */ + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + /** @migrate: Migration helper for save/restore of CCS data */ + struct xe_migrate *migrate; + + struct { + /** @ccs_rw_bb_pool: Pool from which batch buffers are allocated. */ + struct xe_sa_manager *ccs_bb_pool; + } mem; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_types.h index 8300416a6226..24a873c50c49 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_types.h @@ -36,6 +36,12 @@ struct xe_device_vf { /** @migration.gt_flags: Per-GT request flags for VF migration recovery */ unsigned long gt_flags; } migration; + + /** @ccs: VF CCS state data */ + struct { + /** @ccs.initialized: Initilalization of VF CCS is completed or not */ + bool initialized; + } ccs; }; #endif From 864690cf4dd62482b6dd049d82c509886c904303 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 22 Jul 2025 17:35:05 +0530 Subject: [PATCH 016/169] drm/xe/vf: Attach and detach CCS copy commands with BO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attach CCS read/write copy commands to BO for old and new mem types as NULL -> tt or system -> tt. Detach the CCS read/write copy commands from BO while deleting ttm bo from xe_ttm_bo_delete_mem_notify(). Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Matthew Auld Cc: Michał Winiarski Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250722120506.6483-3-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_bb.c | 35 +++++ drivers/gpu/drm/xe/xe_bb.h | 3 + drivers/gpu/drm/xe/xe_bo.c | 23 ++++ drivers/gpu/drm/xe/xe_bo_types.h | 3 + drivers/gpu/drm/xe/xe_migrate.c | 148 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_migrate.h | 7 + drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 67 ++++++++++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h | 3 + drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h | 8 ++ 9 files changed, 297 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c index 5ce0e26822f2..feb6e013dc38 100644 --- a/drivers/gpu/drm/xe/xe_bb.c +++ b/drivers/gpu/drm/xe/xe_bb.c @@ -60,6 +60,41 @@ err: return ERR_PTR(err); } +struct xe_bb *xe_bb_ccs_new(struct xe_gt *gt, u32 dwords, + enum xe_sriov_vf_ccs_rw_ctxs ctx_id) +{ + struct xe_bb *bb = kmalloc(sizeof(*bb), GFP_KERNEL); + struct xe_tile *tile = gt_to_tile(gt); + struct xe_sa_manager *bb_pool; + int err; + + if (!bb) + return ERR_PTR(-ENOMEM); + /* + * We need to allocate space for the requested number of dwords & + * one additional MI_BATCH_BUFFER_END dword. Since the whole SA + * is submitted to HW, we need to make sure that the last instruction + * is not over written when the last chunk of SA is allocated for BB. + * So, this extra DW acts as a guard here. + */ + + bb_pool = tile->sriov.vf.ccs[ctx_id].mem.ccs_bb_pool; + bb->bo = xe_sa_bo_new(bb_pool, 4 * (dwords + 1)); + + if (IS_ERR(bb->bo)) { + err = PTR_ERR(bb->bo); + goto err; + } + + bb->cs = xe_sa_bo_cpu_addr(bb->bo); + bb->len = 0; + + return bb; +err: + kfree(bb); + return ERR_PTR(err); +} + static struct xe_sched_job * __xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 *addr) { diff --git a/drivers/gpu/drm/xe/xe_bb.h b/drivers/gpu/drm/xe/xe_bb.h index b5cc65506696..2a8adc9a6dee 100644 --- a/drivers/gpu/drm/xe/xe_bb.h +++ b/drivers/gpu/drm/xe/xe_bb.h @@ -13,8 +13,11 @@ struct dma_fence; struct xe_gt; struct xe_exec_queue; struct xe_sched_job; +enum xe_sriov_vf_ccs_rw_ctxs; struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm); +struct xe_bb *xe_bb_ccs_new(struct xe_gt *gt, u32 dwords, + enum xe_sriov_vf_ccs_rw_ctxs ctx_id); struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb); struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q, diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 00ce067d5fd3..ffca1cea5585 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -33,6 +33,7 @@ #include "xe_pxp.h" #include "xe_res_cursor.h" #include "xe_shrinker.h" +#include "xe_sriov_vf_ccs.h" #include "xe_trace_bo.h" #include "xe_ttm_stolen_mgr.h" #include "xe_vm.h" @@ -965,6 +966,20 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, dma_fence_put(fence); xe_pm_runtime_put(xe); + /* + * CCS meta data is migrated from TT -> SMEM. So, let us detach the + * BBs from BO as it is no longer needed. + */ + if (IS_VF_CCS_BB_VALID(xe, bo) && old_mem_type == XE_PL_TT && + new_mem->mem_type == XE_PL_SYSTEM) + xe_sriov_vf_ccs_detach_bo(bo); + + if (IS_SRIOV_VF(xe) && + ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || + (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && + handle_system_ccs) + ret = xe_sriov_vf_ccs_attach_bo(bo); + out: if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) && ttm_bo->ttm) { @@ -975,6 +990,9 @@ out: if (timeout < 0) ret = timeout; + if (IS_VF_CCS_BB_VALID(xe, bo)) + xe_sriov_vf_ccs_detach_bo(bo); + xe_tt_unmap_sg(xe, ttm_bo->ttm); } @@ -1502,9 +1520,14 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) { + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + if (!xe_bo_is_xe_bo(ttm_bo)) return; + if (IS_VF_CCS_BB_VALID(ttm_to_xe_device(ttm_bo->bdev), bo)) + xe_sriov_vf_ccs_detach_bo(bo); + /* * Object is idle and about to be destroyed. Release the * dma-buf attachment. diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 57d34698139e..cf604adc13a3 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -77,6 +77,9 @@ struct xe_bo { /** @ccs_cleared */ bool ccs_cleared; + /** @bb_ccs_rw: BB instructions of CCS read/write. Valid only for VF */ + struct xe_bb *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT]; + /** * @cpu_caching: CPU caching mode. Currently only used for userspace * objects. Exceptions are system memory on DGFX, which is always diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 6a80ae6104dd..5a108e34c554 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -30,6 +30,7 @@ #include "xe_mocs.h" #include "xe_pt.h" #include "xe_res_cursor.h" +#include "xe_sa.h" #include "xe_sched_job.h" #include "xe_sync.h" #include "xe_trace_bo.h" @@ -954,6 +955,153 @@ err_sync: return fence; } +/** + * xe_get_migrate_lrc() - Get the LRC from migrate context. + * @migrate: Migrate context. + * + * Return: Pointer to LRC on success, error on failure + */ +struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate) +{ + return migrate->q->lrc[0]; +} + +static int emit_flush_invalidate(struct xe_migrate *m, u32 *dw, int i, + u32 flags) +{ + dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | + MI_FLUSH_IMM_DW | flags; + dw[i++] = lower_32_bits(xe_lrc_start_seqno_ggtt_addr(xe_migrate_lrc(m))) | + MI_FLUSH_DW_USE_GTT; + dw[i++] = upper_32_bits(xe_lrc_start_seqno_ggtt_addr(xe_migrate_lrc(m))); + dw[i++] = MI_NOOP; + dw[i++] = MI_NOOP; + + return i; +} + +/** + * xe_migrate_ccs_rw_copy() - Copy content of TTM resources. + * @m: The migration context. + * @src_bo: The buffer object @src is currently bound to. + * @read_write : Creates BB commands for CCS read/write. + * + * Creates batch buffer instructions to copy CCS metadata from CCS pool to + * memory and vice versa. + * + * This function should only be called for IGPU. + * + * Return: 0 if successful, negative error code on failure. + */ +int xe_migrate_ccs_rw_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write) + +{ + bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; + bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; + struct ttm_resource *src = src_bo->ttm.resource; + struct xe_gt *gt = m->tile->primary_gt; + u32 batch_size, batch_size_allocated; + struct xe_device *xe = gt_to_xe(gt); + struct xe_res_cursor src_it, ccs_it; + u64 size = xe_bo_size(src_bo); + struct xe_bb *bb = NULL; + u64 src_L0, src_L0_ofs; + u32 src_L0_pt; + int err; + + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); + + xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), + PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), + &ccs_it); + + /* Calculate Batch buffer size */ + batch_size = 0; + while (size) { + batch_size += 10; /* Flush + ggtt addr + 2 NOP */ + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + src_L0 = min_t(u64, max_mem_transfer_per_pass(xe), size); + + batch_size += pte_update_size(m, false, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, + &ccs_pt, 0, avail_pts, avail_pts); + xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); + + /* Add copy commands size here */ + batch_size += EMIT_COPY_CCS_DW; + + size -= src_L0; + } + + bb = xe_bb_ccs_new(gt, batch_size, read_write); + if (IS_ERR(bb)) { + drm_err(&xe->drm, "BB allocation failed.\n"); + err = PTR_ERR(bb); + goto err_ret; + } + + batch_size_allocated = batch_size; + size = xe_bo_size(src_bo); + batch_size = 0; + + /* + * Emit PTE and copy commands here. + * The CCS copy command can only support limited size. If the size to be + * copied is more than the limit, divide copy into chunks. So, calculate + * sizes here again before copy command is emitted. + */ + while (size) { + batch_size += 10; /* Flush + ggtt addr + 2 NOP */ + u32 flush_flags = 0; + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + + u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; + + src_L0 = xe_migrate_res_sizes(m, &src_it); + + batch_size += pte_update_size(m, false, src, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + avail_pts); + + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, + &ccs_pt, 0, avail_pts, avail_pts); + xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); + batch_size += EMIT_COPY_CCS_DW; + + emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src); + + emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); + + bb->len = emit_flush_invalidate(m, bb->cs, bb->len, flush_flags); + flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt, + src_L0_ofs, dst_is_pltt, + src_L0, ccs_ofs, true); + bb->len = emit_flush_invalidate(m, bb->cs, bb->len, flush_flags); + + size -= src_L0; + } + + xe_assert(xe, (batch_size_allocated == bb->len)); + src_bo->bb_ccs[read_write] = bb; + + return 0; + +err_ret: + return err; +} + static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch) { diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 74c60f55004a..cd948c7b8dea 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -24,6 +24,8 @@ struct xe_vm; struct xe_vm_pgtable_update; struct xe_vma; +enum xe_sriov_vf_ccs_rw_ctxs; + /** * struct xe_migrate_pt_update_ops - Callbacks for the * xe_migrate_update_pgtables() function. @@ -113,6 +115,11 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, struct ttm_resource *dst, bool copy_only_ccs); +int xe_migrate_ccs_rw_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write); + +struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate); int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, unsigned long offset, void *buf, int len, int write); diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 697eea46659a..6ddeb016b892 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -5,6 +5,7 @@ #include "instructions/xe_mi_commands.h" #include "instructions/xe_gpu_commands.h" +#include "xe_bb.h" #include "xe_bo.h" #include "xe_device.h" #include "xe_migrate.h" @@ -206,3 +207,69 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) err_ret: return err; } + +/** + * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. + * @bo: the &buffer object to which batch buffer commands will be added. + * + * This function shall be called only by VF. It inserts the PTEs and copy + * command instructions in the BO by calling xe_migrate_ccs_rw_copy() + * function. + * + * Returns: 0 if successful, negative error code on failure. + */ +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_migrate *migrate; + struct xe_tile *tile; + struct xe_bb *bb; + int err = 0; + + if (!IS_VF_CCS_READY(xe)) + return 0; + + tile = xe_device_get_root_tile(xe); + + for_each_ccs_rw_ctx(ctx_id) { + bb = bo->bb_ccs[ctx_id]; + /* bb should be NULL here. Assert if not NULL */ + xe_assert(xe, !bb); + + migrate = tile->sriov.vf.ccs[ctx_id].migrate; + err = xe_migrate_ccs_rw_copy(migrate, bo, ctx_id); + } + return err; +} + +/** + * xe_sriov_vf_ccs_detach_bo - Remove CCS read write commands from the BO. + * @bo: the &buffer object from which batch buffer commands will be removed. + * + * This function shall be called only by VF. It removes the PTEs and copy + * command instructions from the BO. Make sure to update the BB with MI_NOOP + * before freeing. + * + * Returns: 0 if successful. + */ +int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_bb *bb; + + if (!IS_VF_CCS_READY(xe)) + return 0; + + for_each_ccs_rw_ctx(ctx_id) { + bb = bo->bb_ccs[ctx_id]; + if (!bb) + continue; + + memset(bb->cs, MI_NOOP, bb->len * sizeof(u32)); + xe_bb_free(bb, NULL); + bo->bb_ccs[ctx_id] = NULL; + } + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h index 5df9ba028d14..5d5e4bd25904 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -7,7 +7,10 @@ #define _XE_SRIOV_VF_CCS_H_ struct xe_device; +struct xe_bo; int xe_sriov_vf_ccs_init(struct xe_device *xe); +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo); +int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); #endif diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h index 6dc279d206ec..e240f3fd18af 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h @@ -27,6 +27,14 @@ enum xe_sriov_vf_ccs_rw_ctxs { XE_SRIOV_VF_CCS_CTX_COUNT }; +#define IS_VF_CCS_BB_VALID(xe, bo) ({ \ + struct xe_device *___xe = (xe); \ + struct xe_bo *___bo = (bo); \ + IS_SRIOV_VF(___xe) && \ + ___bo->bb_ccs[XE_SRIOV_VF_CCS_READ_CTX] && \ + ___bo->bb_ccs[XE_SRIOV_VF_CCS_WRITE_CTX]; \ + }) + struct xe_migrate; struct xe_sa_manager; From 916ee4704a8653910f10c65c2b5d6699dfac5df8 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 22 Jul 2025 17:35:06 +0530 Subject: [PATCH 017/169] drm/xe/vf: Register CCS read/write contexts with Guc Register read write contexts with newly added flags with GUC and enable the context immediately after registration. Re-register the context with Guc when resuming from runtime suspend as soft reset is applied to Guc during xe_pm_runtime_resume(). Make Ring head=tail while unbinding device to avoid issues with VF pause after device is unbinded. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Maarten Lankhorst Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250722120506.6483-4-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_guc_fwif.h | 5 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 34 +++++++++- drivers/gpu/drm/xe/xe_guc_submit.h | 1 + drivers/gpu/drm/xe/xe_migrate.c | 24 ++++---- drivers/gpu/drm/xe/xe_migrate.h | 3 +- drivers/gpu/drm/xe/xe_pm.c | 4 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 92 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h | 1 + drivers/gpu/drm/xe/xe_vm.c | 6 +- 9 files changed, 150 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index 6f57578b07cb..ca9f999d38d1 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -45,6 +45,11 @@ #define GUC_MAX_ENGINE_CLASSES 16 #define GUC_MAX_INSTANCES_PER_CLASS 32 +#define GUC_CONTEXT_NORMAL 0 +#define GUC_CONTEXT_COMPRESSION_SAVE 1 +#define GUC_CONTEXT_COMPRESSION_RESTORE 2 +#define GUC_CONTEXT_COUNT (GUC_CONTEXT_COMPRESSION_RESTORE + 1) + /* Helper for context registration H2G */ struct guc_ctxt_registration_info { u32 flags; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 5f91b9a6ab7d..679ae229cc82 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -542,7 +542,7 @@ static void __register_exec_queue(struct xe_guc *guc, xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); } -static void register_exec_queue(struct xe_exec_queue *q) +static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); @@ -550,6 +550,7 @@ static void register_exec_queue(struct xe_exec_queue *q) struct guc_ctxt_registration_info info; xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT); memset(&info, 0, sizeof(info)); info.context_idx = q->guc->id; @@ -559,6 +560,9 @@ static void register_exec_queue(struct xe_exec_queue *q) info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); info.flags = CONTEXT_REGISTRATION_FLAG_KMD; + if (ctx_type != GUC_CONTEXT_NORMAL) + info.flags |= BIT(ctx_type); + if (xe_exec_queue_is_parallel(q)) { u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); struct iosys_map map = xe_lrc_parallel_map(lrc); @@ -761,7 +765,7 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) { if (!exec_queue_registered(q)) - register_exec_queue(q); + register_exec_queue(q, GUC_CONTEXT_NORMAL); if (!lr) /* LR jobs are emitted in the exec IOCTL */ q->ring_ops->emit_job(job); submit_exec_queue(q); @@ -2383,6 +2387,32 @@ static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) xe_guc_exec_queue_snapshot_free(snapshot); } +/** + * xe_guc_register_exec_queue - Register exec queue for a given context type. + * @q: Execution queue + * @ctx_type: Type of the context + * + * This function registers the execution queue with the guc. Special context + * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE + * are only applicable for IGPU and in the VF. + * Submits the execution queue to GUC after registering it. + * + * Returns - None. + */ +void xe_guc_register_exec_queue(struct xe_exec_queue *q, int ctx_type) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + + xe_assert(xe, IS_SRIOV_VF(xe)); + xe_assert(xe, !IS_DGFX(xe)); + xe_assert(xe, (ctx_type > GUC_CONTEXT_NORMAL && + ctx_type < GUC_CONTEXT_COUNT)); + + register_exec_queue(q, ctx_type); + enable_scheduling(q); +} + /** * xe_guc_submit_print - GuC Submit Print. * @guc: GuC. diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 9b71a986c6ca..8f64e799283b 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -39,5 +39,6 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot); void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); +void xe_guc_register_exec_queue(struct xe_exec_queue *q, int ctx_type); #endif diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 5a108e34c554..0a61fb8b123a 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -86,19 +86,6 @@ struct xe_migrate { */ #define MAX_PTE_PER_SDI 0x1FEU -/** - * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue. - * @tile: The tile. - * - * Returns the default migrate exec queue of this tile. - * - * Return: The default migrate exec queue - */ -struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile) -{ - return tile->migrate->q; -} - static void xe_migrate_fini(void *arg) { struct xe_migrate *m = arg; @@ -1102,6 +1089,17 @@ err_ret: return err; } +/** + * xe_get_migrate_exec_queue() - Get the execution queue from migrate context. + * @migrate: Migrate context. + * + * Return: Pointer to execution queue on success, error on failure + */ +struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate) +{ + return migrate->q; +} + static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch) { diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index cd948c7b8dea..f4dac270dba3 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -120,6 +120,7 @@ int xe_migrate_ccs_rw_copy(struct xe_migrate *m, enum xe_sriov_vf_ccs_rw_ctxs read_write); struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate); +struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate); int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, unsigned long offset, void *buf, int len, int write); @@ -140,6 +141,4 @@ xe_migrate_update_pgtables(struct xe_migrate *m, struct xe_migrate_pt_update *pt_update); void xe_migrate_wait(struct xe_migrate *m); - -struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile); #endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index e279b47ba03b..ed29791beb7c 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -23,6 +23,7 @@ #include "xe_irq.h" #include "xe_pcode.h" #include "xe_pxp.h" +#include "xe_sriov_vf_ccs.h" #include "xe_trace.h" #include "xe_wa.h" @@ -558,6 +559,9 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_pxp_pm_resume(xe->pxp); + if (IS_SRIOV_VF(xe)) + xe_sriov_vf_ccs_register_context(xe); + out: xe_rpm_lockmap_release(xe); xe_pm_write_callback_task(xe, NULL); diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 6ddeb016b892..af43e04179aa 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -8,6 +8,9 @@ #include "xe_bb.h" #include "xe_bo.h" #include "xe_device.h" +#include "xe_exec_queue_types.h" +#include "xe_guc_submit.h" +#include "xe_lrc.h" #include "xe_migrate.h" #include "xe_sa.h" #include "xe_sriov_printk.h" @@ -163,6 +166,85 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_tile_vf_ccs *ctx) return 0; } +static void ccs_rw_update_ring(struct xe_tile_vf_ccs *ctx) +{ + struct xe_lrc *lrc = xe_migrate_lrc(ctx->migrate); + u64 addr = ctx->mem.ccs_bb_pool->gpu_addr; + u32 dw[10], i = 0; + + dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; + dw[i++] = MI_BATCH_BUFFER_START | XE_INSTR_NUM_DW(3); + dw[i++] = lower_32_bits(addr); + dw[i++] = upper_32_bits(addr); + dw[i++] = MI_NOOP; + dw[i++] = MI_NOOP; + + xe_lrc_write_ring(lrc, dw, i * sizeof(u32)); + xe_lrc_set_ring_tail(lrc, lrc->ring.tail); +} + +static int register_save_restore_context(struct xe_migrate *m, + enum xe_sriov_vf_ccs_rw_ctxs ctx_id) +{ + int err = -EINVAL; + int ctx_type; + + switch (ctx_id) { + case XE_SRIOV_VF_CCS_READ_CTX: + ctx_type = GUC_CONTEXT_COMPRESSION_SAVE; + break; + case XE_SRIOV_VF_CCS_WRITE_CTX: + ctx_type = GUC_CONTEXT_COMPRESSION_RESTORE; + break; + default: + return err; + } + + xe_guc_register_exec_queue(xe_migrate_exec_queue(m), ctx_type); + return 0; +} + +/** + * xe_sriov_vf_ccs_register_context - Register read/write contexts with guc. + * @xe: the &xe_device to register contexts on. + * + * This function registers read and write contexts with Guc. Re-registration + * is needed whenever resuming from pm runtime suspend. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_sriov_vf_ccs_register_context(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_tile_vf_ccs *ctx; + int err; + + if (!IS_VF_CCS_READY(xe)) + return 0; + + for_each_ccs_rw_ctx(ctx_id) { + ctx = &tile->sriov.vf.ccs[ctx_id]; + err = register_save_restore_context(ctx->migrate, ctx_id); + if (err) + return err; + } + + return err; +} + +static void xe_sriov_vf_ccs_fini(void *arg) +{ + struct xe_tile_vf_ccs *ctx = arg; + struct xe_lrc *lrc = xe_migrate_lrc(ctx->migrate); + + /* + * Make TAIL = HEAD in the ring so that no issues are seen if Guc + * submits this context to HW on VF pause after unbinding device. + */ + xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc)); +} + /** * xe_sriov_vf_ccs_init - Setup LRCA for save & restore. * @xe: the &xe_device to start recovery on @@ -198,6 +280,16 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) err = alloc_bb_pool(tile, ctx); if (err) goto err_ret; + + ccs_rw_update_ring(ctx); + + err = register_save_restore_context(ctx->migrate, ctx_id); + if (err) + goto err_ret; + + err = devm_add_action_or_reset(xe->drm.dev, + xe_sriov_vf_ccs_fini, + ctx); } xe->sriov.vf.ccs.initialized = 1; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h index 5d5e4bd25904..1f1baf685fec 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -12,5 +12,6 @@ struct xe_bo; int xe_sriov_vf_ccs_init(struct xe_device *xe); int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo); int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); +int xe_sriov_vf_ccs_register_context(struct xe_device *xe); #endif diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 2035604121e6..432ea325677d 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -953,7 +953,7 @@ struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_ma for_each_tile(tile, vm->xe, id) { vops.pt_update_ops[id].wait_vm_bookkeep = true; vops.pt_update_ops[tile->id].q = - xe_tile_migrate_exec_queue(tile); + xe_migrate_exec_queue(tile->migrate); } err = xe_vm_ops_add_rebind(&vops, vma, tile_mask); @@ -1043,7 +1043,7 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, for_each_tile(tile, vm->xe, id) { vops.pt_update_ops[id].wait_vm_bookkeep = true; vops.pt_update_ops[tile->id].q = - xe_tile_migrate_exec_queue(tile); + xe_migrate_exec_queue(tile->migrate); } err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask); @@ -1126,7 +1126,7 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, for_each_tile(tile, vm->xe, id) { vops.pt_update_ops[id].wait_vm_bookkeep = true; vops.pt_update_ops[tile->id].q = - xe_tile_migrate_exec_queue(tile); + xe_migrate_exec_queue(tile->migrate); } err = xe_vm_ops_add_range_unbind(&vops, range); From 4d3bbe9dd28c0a4ca119e4b8823c5f5e9cb3ff90 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Tue, 22 Jul 2025 12:52:08 -0700 Subject: [PATCH 018/169] drm/xe: Fix build without debugfs When CONFIG_DEBUG_FS is off, drivers/gpu/drm/xe/xe_gt_debugfs.o is not built and build fails on some setups with: ld: drivers/gpu/drm/xe/xe_gt.o: in function `xe_fault_inject_gt_reset': drivers/gpu/drm/xe/xe_gt.h:27:(.text+0x1659): undefined reference to `gt_reset_failure' ld: drivers/gpu/drm/xe/xe_gt.h:27:(.text+0x1c16): undefined reference to `gt_reset_failure' collect2: error: ld returned 1 exit status Do not use the gt_reset_failure attribute if debugfs is not enabled. Fixes: 8f3013e0b222 ("drm/xe: Introduce fault injection for gt reset") Cc: Himal Prasad Ghimiray Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Himal Prasad Ghimiray Link: https://lore.kernel.org/r/20250722-xe-fix-build-fault-v1-1-157384d50987@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_gt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h index e9ccab8aedbe..41880979f4de 100644 --- a/drivers/gpu/drm/xe/xe_gt.h +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -24,7 +24,7 @@ extern struct fault_attr gt_reset_failure; static inline bool xe_fault_inject_gt_reset(void) { - return should_fail(>_reset_failure, 1); + return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(>_reset_failure, 1); } struct xe_gt *xe_gt_alloc(struct xe_tile *tile); From a2b461bd6f3b36bded0a74178dec0e58e4714d3d Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 20:26:16 +0200 Subject: [PATCH 019/169] drm/xe/pf: Enable SR-IOV PF mode by default We already claim official support for SR-IOV PF/VF modes on PTL and BMG platforms, but by default we start the Xe driver on those platforms in non-virtualized mode (native) since we still have max_vfs modparam set to disable creation of the VFs. It's time to let the Xe driver support SR-IOV PF mode by default. We were already testing this on our CI, which was relying on the patch that was enabling it for CONFIG_DRM_XE_DEBUG used by our CI. Signed-off-by: Michal Wajdeczko Cc: Thomas Hellstrom Cc: Lucas De Marchi Cc: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250722182618.30811-3-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_module.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index 107ffe87808c..d9391bd08194 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -27,6 +27,8 @@ #define DEFAULT_PROBE_DISPLAY true #define DEFAULT_VRAM_BAR_SIZE 0 #define DEFAULT_FORCE_PROBE CONFIG_DRM_XE_FORCE_PROBE +#define DEFAULT_MAX_VFS ~0 +#define DEFAULT_MAX_VFS_STR "unlimited" #define DEFAULT_WEDGED_MODE 1 #define DEFAULT_SVM_NOTIFIER_SIZE 512 @@ -34,6 +36,9 @@ struct xe_modparam xe_modparam = { .probe_display = DEFAULT_PROBE_DISPLAY, .guc_log_level = DEFAULT_GUC_LOG_LEVEL, .force_probe = DEFAULT_FORCE_PROBE, +#ifdef CONFIG_PCI_IOV + .max_vfs = DEFAULT_MAX_VFS, +#endif .wedged_mode = DEFAULT_WEDGED_MODE, .svm_notifier_size = DEFAULT_SVM_NOTIFIER_SIZE, /* the rest are 0 by default */ @@ -79,7 +84,8 @@ MODULE_PARM_DESC(force_probe, module_param_named(max_vfs, xe_modparam.max_vfs, uint, 0400); MODULE_PARM_DESC(max_vfs, "Limit number of Virtual Functions (VFs) that could be managed. " - "(0 = no VFs [default]; N = allow up to N VFs)"); + "(0=no VFs; N=allow up to N VFs " + "[default=" DEFAULT_MAX_VFS_STR "])"); #endif module_param_named_unsafe(wedged_mode, xe_modparam.wedged_mode, int, 0600); From 2e761039985271a69bf6eada8f236701f8594638 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 20:26:17 +0200 Subject: [PATCH 020/169] drm/xe: Enable SR-IOV for ADL/ATSM We were already testing those two platforms for a while on our CI, but enabling flag (has_sriov) was only available on the topic branch and only for builds with CONFIG_DRM_XE_DEBUG config. Since those two platforms are guarded by the another enabling flag (require_force_probe) and we believe our SR-IOV support for them is at sufficient level to start enjoying the feature, turn on the SR-IOV enabling flag unconditionally. Suggested-by: Lucas De Marchi Signed-off-by: Michal Wajdeczko Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250722182618.30811-4-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_pci.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index f64942737a0b..c672af43d361 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -193,6 +193,7 @@ static const struct xe_device_desc adl_s_desc = { .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, .max_gt_per_tile = 1, .require_force_probe = true, .subplatforms = (const struct xe_subplatform_desc[]) { @@ -210,6 +211,7 @@ static const struct xe_device_desc adl_p_desc = { .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, .max_gt_per_tile = 1, .require_force_probe = true, .subplatforms = (const struct xe_subplatform_desc[]) { @@ -225,6 +227,7 @@ static const struct xe_device_desc adl_n_desc = { .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, .max_gt_per_tile = 1, .require_force_probe = true, }; @@ -270,6 +273,7 @@ static const struct xe_device_desc ats_m_desc = { DG2_FEATURES, .has_display = false, + .has_sriov = true, }; static const struct xe_device_desc dg2_desc = { From 6983ea9cd720fdd409b4944caf9605731323bb8d Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 20:26:18 +0200 Subject: [PATCH 021/169] drm/xe: Enable SR-IOV for TGL While we don't have official CI SR-IOV coverage for the Tigerlake platforms, we were using this platform for the feature enabling and Xe driver already has all required changes to support it. Since TGL platforms are guarded by the xe.require_force_probe flag enable SR-IOV feature on them, like we recently did for ADL/ATSM. Suggested-by: Lucas De Marchi Signed-off-by: Michal Wajdeczko Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250722182618.30811-5-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index c672af43d361..053faa7f9b01 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -169,6 +169,7 @@ static const struct xe_device_desc tgl_desc = { .dma_mask_size = 39, .has_display = true, .has_llc = true, + .has_sriov = true, .max_gt_per_tile = 1, .require_force_probe = true, }; From 538b27a09af974f2c5e7d358a24af7a722fa4bfc Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 23 Jul 2025 15:30:15 +0200 Subject: [PATCH 022/169] drm/xe: Make GGTT TLB invalidation failure message GT oriented GGTT TLB invalidation is performed on the specific GT, thus any failure message shall be also GT specific. And to help investigate any unexpected failures, promote message from warn level to WARN to get full call stack of this unlikely case. Signed-off-by: Michal Wajdeczko Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250723133015.206601-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_ggtt.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 29d4d3f51da1..249713139f69 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -439,8 +439,7 @@ static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) return; err = xe_gt_tlb_invalidation_ggtt(gt); - if (err) - drm_warn(>_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err); + xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err)); } static void xe_ggtt_invalidate(struct xe_ggtt *ggtt) From 159afd92bae8153bdd8d8b34aea0d463fe19c978 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 23 Jul 2025 19:56:39 +0200 Subject: [PATCH 023/169] drm/xe/guc: Clear whole g2h_fence during initialization The struct g2h_fence must be explicitly initializated using the g2h_fence_init() function to avoid trash values in its members, but we missed to update this helper function with the new member. To fix that and avoid any future mistakes, memset the whole struct first, then update remaining non-zero members. Fixes: 94de94d24ea8 ("drm/xe/guc: Cancel ongoing H2G requests when stopping CT") Signed-off-by: Michal Wajdeczko Cc: Matthew Brost Cc: Lukasz Laguna Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250723175639.206875-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_guc_ct.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index b6acccfcd351..3f4e6a46ff16 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -95,12 +95,8 @@ struct g2h_fence { static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) { + memset(g2h_fence, 0, sizeof(*g2h_fence)); g2h_fence->response_buffer = response_buffer; - g2h_fence->response_data = 0; - g2h_fence->response_len = 0; - g2h_fence->fail = false; - g2h_fence->retry = false; - g2h_fence->done = false; g2h_fence->seqno = ~0x0; } From a313d9059f00adefdadc6c5612c104bbd78a4808 Mon Sep 17 00:00:00 2001 From: Nitin Gote Date: Wed, 23 Jul 2025 19:40:39 +0530 Subject: [PATCH 024/169] drm/xe: Rename MCFG_MCR_SELECTOR to STEER_SEMAPHORE The register at offset 0xfd0 was incorrectly named MCFG_MCR_SELECTOR, likely copied from i915. According to the hardware specification (Bspec), this register is actually called STEER_SEMAPHORE. Rename the register definition and update its usage in xe_gt_mcr.c to match the official hardware documentation. No functional changes. v2: Add Bspec reference (Tejas) Bspec: 67113 Signed-off-by: Nitin Gote Reviewed-by: Tejas Upadhyay Link: https://lore.kernel.org/r/20250723141039.3848390-1-nitin.r.gote@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/regs/xe_gt_regs.h | 2 +- drivers/gpu/drm/xe/xe_gt_mcr.c | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 5cd5ab8529c5..f96b2e2b3064 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -42,7 +42,7 @@ #define FORCEWAKE_ACK_GSC XE_REG(0xdf8) #define FORCEWAKE_ACK_GT_MTL XE_REG(0xdfc) -#define MCFG_MCR_SELECTOR XE_REG(0xfd0) +#define STEER_SEMAPHORE XE_REG(0xfd0) #define MTL_MCR_SELECTOR XE_REG(0xfd4) #define SF_MCR_SELECTOR XE_REG(0xfd8) #define MCR_SELECTOR XE_REG(0xfdc) diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c index 64a2f0d6aaf9..683ac021a06d 100644 --- a/drivers/gpu/drm/xe/xe_gt_mcr.c +++ b/drivers/gpu/drm/xe/xe_gt_mcr.c @@ -46,8 +46,6 @@ * MCR registers are not available on Virtual Function (VF). */ -#define STEER_SEMAPHORE XE_REG(0xFD0) - static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) { return reg_mcr.__reg; @@ -533,7 +531,7 @@ void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); - xe_mmio_write32(>->mmio, MCFG_MCR_SELECTOR, steer_val); + xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val); xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); /* * For GAM registers, all reads should be directed to instance 1 From a98cdd979c2b529363be502cb057aea79e65fbf7 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Thu, 24 Jul 2025 14:16:58 +0100 Subject: [PATCH 025/169] drm/xe: Use emit_flush_imm_ggtt helper instead of open coding Helper is already there so lets just use it. Signed-off-by: Tvrtko Ursulin Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250724131711.74291-2-tvrtko.ursulin@igalia.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_ring_ops.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 7b50c7c1ee21..e8f22ec5f9af 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -415,11 +415,9 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); - dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags | - MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW; - dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT; - dw[i++] = 0; - dw[i++] = seqno; /* value */ + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, + MI_INVALIDATE_TLB | job->migrate_flush_flags, + dw, i); i = emit_user_interrupt(dw, i); From d72779c29d82c6e371cea8b427550bd6923c2577 Mon Sep 17 00:00:00 2001 From: Sk Anirban Date: Wed, 16 Jul 2025 15:46:23 +0530 Subject: [PATCH 026/169] drm/xe/ptl: Apply Wa_16026007364 As part of this WA GuC will save and restore value of two XE3_Media control registers that were not included in the HW power context. Signed-off-by: Sk Anirban Reviewed-by: Daniele Ceraolo Spurio Link: https://lore.kernel.org/r/20250716101622.3421480-2-sk.anirban@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/abi/guc_klvs_abi.h | 1 + drivers/gpu/drm/xe/xe_guc_ads.c | 35 +++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_wa_oob.rules | 1 + 3 files changed, 37 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index 0366a9da5977..31dbfeee289e 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -396,6 +396,7 @@ enum xe_guc_klv_ids { GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET = 0x9009, GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO = 0x900a, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH = 0x900b, + GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG = 0x900c, }; #endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 131cfc56be00..8ff8626227ae 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -284,6 +284,35 @@ static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) return total_size; } +static void guc_waklv_enable_two_word(struct xe_guc_ads *ads, + enum xe_guc_klv_ids klv_id, + u32 value1, + u32 value2, + u32 *offset, u32 *remain) +{ + u32 size; + u32 klv_entry[] = { + /* 16:16 key/length */ + FIELD_PREP(GUC_KLV_0_KEY, klv_id) | + FIELD_PREP(GUC_KLV_0_LEN, 2), + value1, + value2, + /* 2 dword data */ + }; + + size = sizeof(klv_entry); + + if (*remain < size) { + drm_warn(&ads_to_xe(ads)->drm, + "w/a klv buffer too small to add klv id %d\n", klv_id); + } else { + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, + klv_entry, size); + *offset += size; + *remain -= size; + } +} + static void guc_waklv_enable_one_word(struct xe_guc_ads *ads, enum xe_guc_klv_ids klv_id, u32 value, @@ -381,6 +410,12 @@ static void guc_waklv_init(struct xe_guc_ads *ads) guc_waklv_enable_simple(ads, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH, &offset, &remain); + if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_WA(gt, 16026007364)) + guc_waklv_enable_two_word(ads, + GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG, + 0x0, + 0xF, + &offset, &remain); size = guc_ads_waklv_size(ads) - remain; if (!size) diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index e990f20eccfe..46ad72bb1f8d 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -72,3 +72,4 @@ no_media_l3 MEDIA_VERSION(3000) 14022085890 GRAPHICS_VERSION(2001) 15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) +16026007364 MEDIA_VERSION(3000) From c3ead4ecfc4c02806aa1c202a595ee3ecb2c039a Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:10 -0700 Subject: [PATCH 027/169] drm/xe: Explicitly mark migration queues with flag Rather than inferring if an exec queue is a migration queue for a flag, explicitly mark migration queues with a flag. Signed-off-by: Matthew Brost Reviewed-by: Francois Dugast Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-2-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 ++ drivers/gpu/drm/xe/xe_migrate.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index cc1cffb5c87f..abdf4a57e6e2 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -87,6 +87,8 @@ struct xe_exec_queue { #define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(4) /* flag to indicate low latency hint to guc */ #define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(5) +/* for migration (kernel copy, clear, bind) jobs */ +#define EXEC_QUEUE_FLAG_MIGRATE BIT(6) /** * @flags: flags for this exec queue, should statically setup aside from ban diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 0a61fb8b123a..ed04c2b368dc 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -439,12 +439,14 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_PERMANENT | - EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); + EXEC_QUEUE_FLAG_HIGH_PRIORITY | + EXEC_QUEUE_FLAG_MIGRATE, 0); } else { m->q = xe_exec_queue_create_class(xe, primary_gt, vm, XE_ENGINE_CLASS_COPY, EXEC_QUEUE_FLAG_KERNEL | - EXEC_QUEUE_FLAG_PERMANENT, 0); + EXEC_QUEUE_FLAG_PERMANENT | + EXEC_QUEUE_FLAG_MIGRATE, 0); } if (IS_ERR(m->q)) { xe_vm_close_and_put(vm); From 69f187d446c94f66d413819f16e2725b7c31f378 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:11 -0700 Subject: [PATCH 028/169] drm/xe: Add generic dependecy jobs / scheduler Add generic dependecy jobs / scheduler which serves as wrapper for DRM scheduler. Useful when we want delay a generic operation until a dma-fence signals. Existing use cases could be destroying of resources based fences / dma-resv, the preempt rebind worker, and pipelined GT TLB invalidations. Written in such a way it could be moved to DRM subsystem if needed. v3: - Remove unnecessary cast (Staurt) Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-3-matthew.brost@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_dep_job_types.h | 29 ++++++ drivers/gpu/drm/xe/xe_dep_scheduler.c | 143 ++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_dep_scheduler.h | 21 ++++ 4 files changed, 194 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_dep_job_types.h create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.c create mode 100644 drivers/gpu/drm/xe/xe_dep_scheduler.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index b046e1b8d726..6e1fd6560557 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -35,6 +35,7 @@ $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe xe-y += xe_bb.o \ xe_bo.o \ xe_bo_evict.o \ + xe_dep_scheduler.o \ xe_devcoredump.o \ xe_device.o \ xe_device_sysfs.o \ diff --git a/drivers/gpu/drm/xe/xe_dep_job_types.h b/drivers/gpu/drm/xe/xe_dep_job_types.h new file mode 100644 index 000000000000..c6a484f24c8c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_job_types.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_DEP_JOB_TYPES_H_ +#define _XE_DEP_JOB_TYPES_H_ + +#include + +struct xe_dep_job; + +/** struct xe_dep_job_ops - Generic Xe dependency job operations */ +struct xe_dep_job_ops { + /** @run_job: Run generic Xe dependency job */ + struct dma_fence *(*run_job)(struct xe_dep_job *job); + /** @free_job: Free generic Xe dependency job */ + void (*free_job)(struct xe_dep_job *job); +}; + +/** struct xe_dep_job - Generic dependency Xe job */ +struct xe_dep_job { + /** @drm: base DRM scheduler job */ + struct drm_sched_job drm; + /** @ops: dependency job operations */ + const struct xe_dep_job_ops *ops; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c new file mode 100644 index 000000000000..9bd3bfd2e526 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include + +#include "xe_dep_job_types.h" +#include "xe_dep_scheduler.h" +#include "xe_device_types.h" + +/** + * DOC: Xe Dependency Scheduler + * + * The Xe dependency scheduler is a simple wrapper built around the DRM + * scheduler to execute jobs once their dependencies are resolved (i.e., all + * input fences specified as dependencies are signaled). The jobs that are + * executed contain virtual functions to run (execute) and free the job, + * allowing a single dependency scheduler to handle jobs performing different + * operations. + * + * Example use cases include deferred resource freeing, TLB invalidations after + * bind jobs, etc. + */ + +/** struct xe_dep_scheduler - Generic Xe dependency scheduler */ +struct xe_dep_scheduler { + /** @sched: DRM GPU scheduler */ + struct drm_gpu_scheduler sched; + /** @entity: DRM scheduler entity */ + struct drm_sched_entity entity; + /** @rcu: For safe freeing of exported dma fences */ + struct rcu_head rcu; +}; + +static struct dma_fence *xe_dep_scheduler_run_job(struct drm_sched_job *drm_job) +{ + struct xe_dep_job *dep_job = + container_of(drm_job, typeof(*dep_job), drm); + + return dep_job->ops->run_job(dep_job); +} + +static void xe_dep_scheduler_free_job(struct drm_sched_job *drm_job) +{ + struct xe_dep_job *dep_job = + container_of(drm_job, typeof(*dep_job), drm); + + dep_job->ops->free_job(dep_job); +} + +static const struct drm_sched_backend_ops sched_ops = { + .run_job = xe_dep_scheduler_run_job, + .free_job = xe_dep_scheduler_free_job, +}; + +/** + * xe_dep_scheduler_create() - Generic Xe dependency scheduler create + * @xe: Xe device + * @submit_wq: Submit workqueue struct (can be NULL) + * @name: Name of dependency scheduler + * @job_limit: Max dependency jobs that can be scheduled + * + * Create a generic Xe dependency scheduler and initialize internal DRM + * scheduler objects. + * + * Return: Generic Xe dependency scheduler object on success, ERR_PTR failure + */ +struct xe_dep_scheduler * +xe_dep_scheduler_create(struct xe_device *xe, + struct workqueue_struct *submit_wq, + const char *name, u32 job_limit) +{ + struct xe_dep_scheduler *dep_scheduler; + struct drm_gpu_scheduler *sched; + const struct drm_sched_init_args args = { + .ops = &sched_ops, + .submit_wq = submit_wq, + .num_rqs = 1, + .credit_limit = job_limit, + .timeout = MAX_SCHEDULE_TIMEOUT, + .name = name, + .dev = xe->drm.dev, + }; + int err; + + dep_scheduler = kzalloc(sizeof(*dep_scheduler), GFP_KERNEL); + if (!dep_scheduler) + return ERR_PTR(-ENOMEM); + + err = drm_sched_init(&dep_scheduler->sched, &args); + if (err) + goto err_free; + + sched = &dep_scheduler->sched; + err = drm_sched_entity_init(&dep_scheduler->entity, 0, &sched, 1, NULL); + if (err) + goto err_sched; + + init_rcu_head(&dep_scheduler->rcu); + + return dep_scheduler; + +err_sched: + drm_sched_fini(&dep_scheduler->sched); +err_free: + kfree(dep_scheduler); + + return ERR_PTR(err); +} + +/** + * xe_dep_scheduler_fini() - Generic Xe dependency scheduler finalize + * @dep_scheduler: Generic Xe dependency scheduler object + * + * Finalize internal DRM scheduler objects and free generic Xe dependency + * scheduler object + */ +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler) +{ + drm_sched_entity_fini(&dep_scheduler->entity); + drm_sched_fini(&dep_scheduler->sched); + /* + * RCU free due sched being exported via DRM scheduler fences + * (timeline name). + */ + kfree_rcu(dep_scheduler, rcu); +} + +/** + * xe_dep_scheduler_entity() - Retrieve a generic Xe dependency scheduler + * DRM scheduler entity + * @dep_scheduler: Generic Xe dependency scheduler object + * + * Return: The generic Xe dependency scheduler's DRM scheduler entity + */ +struct drm_sched_entity * +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler) +{ + return &dep_scheduler->entity; +} diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h new file mode 100644 index 000000000000..853961eec64b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +struct drm_sched_entity; +struct workqueue_struct; +struct xe_dep_scheduler; +struct xe_device; + +struct xe_dep_scheduler * +xe_dep_scheduler_create(struct xe_device *xe, + struct workqueue_struct *submit_wq, + const char *name, u32 job_limit); + +void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler); + +struct drm_sched_entity * +xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler); From ada51219489f53f9f35833c3d0d8a4fe3f4c5705 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:12 -0700 Subject: [PATCH 029/169] drm/xe: Create ordered workqueue for GT TLB invalidation jobs No sense to schedule GT TLB invalidation jobs in parallel which target the same GT given these all contend on the same lock, create ordered workqueue for GT TLB invalidation jobs. v3: - Fix type in commmit message (Stuart) Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-4-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 8 ++++++++ drivers/gpu/drm/xe/xe_gt_types.h | 2 ++ 2 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c index 086c12ee3d9d..02f0bb92d6e0 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c @@ -3,6 +3,8 @@ * Copyright © 2023 Intel Corporation */ +#include + #include "xe_gt_tlb_invalidation.h" #include "abi/guc_actions_abi.h" @@ -123,6 +125,12 @@ int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt) INIT_DELAYED_WORK(>->tlb_invalidation.fence_tdr, xe_gt_tlb_fence_timeout); + gt->tlb_invalidation.job_wq = + drmm_alloc_ordered_workqueue(>_to_xe(gt)->drm, "gt-tbl-inval-job-wq", + WQ_MEM_RECLAIM); + if (IS_ERR(gt->tlb_invalidation.job_wq)) + return PTR_ERR(gt->tlb_invalidation.job_wq); + return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 96344c604726..dfd4a16da5f0 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -210,6 +210,8 @@ struct xe_gt { * xe_gt_tlb_fence_timeout after the timeut interval is over. */ struct delayed_work fence_tdr; + /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */ + struct workqueue_struct *job_wq; /** @tlb_invalidation.lock: protects TLB invalidation fences */ spinlock_t lock; } tlb_invalidation; From 535c445eb94c9abce3350aa61424fc34b8138ee8 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:13 -0700 Subject: [PATCH 030/169] drm/xe: Add dependency scheduler for GT TLB invalidations to bind queues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a generic dependency scheduler for GT TLB invalidations, used to schedule jobs that issue GT TLB invalidations to bind queues. v2: - Use shared GT TLB invalidation queue for dep scheduler - Break allocation of dep scheduler into its own function - Add define for max number tlb invalidations - Skip media if not present Suggested-by: Thomas Hellström Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-5-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 48 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_exec_queue_types.h | 13 +++++++ 2 files changed, 61 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 8991b4aed440..d5958518ef00 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -12,6 +12,7 @@ #include #include +#include "xe_dep_scheduler.h" #include "xe_device.h" #include "xe_gt.h" #include "xe_hw_engine_class_sysfs.h" @@ -39,6 +40,12 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue static void __xe_exec_queue_free(struct xe_exec_queue *q) { + int i; + + for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) + if (q->tlb_inval[i].dep_scheduler) + xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); + if (xe_exec_queue_uses_pxp(q)) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); if (q->vm) @@ -50,6 +57,39 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) kfree(q); } +static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_tile *tile = gt_to_tile(q->gt); + int i; + + for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { + struct xe_dep_scheduler *dep_scheduler; + struct xe_gt *gt; + struct workqueue_struct *wq; + + if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) + gt = tile->primary_gt; + else + gt = tile->media_gt; + + if (!gt) + continue; + + wq = gt->tlb_invalidation.job_wq; + +#define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ + dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, + MAX_TLB_INVAL_JOBS); + if (IS_ERR(dep_scheduler)) + return PTR_ERR(dep_scheduler); + + q->tlb_inval[i].dep_scheduler = dep_scheduler; + } +#undef MAX_TLB_INVAL_JOBS + + return 0; +} + static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, @@ -94,6 +134,14 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, else q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; + if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { + err = alloc_dep_schedulers(xe, q); + if (err) { + __xe_exec_queue_free(q); + return ERR_PTR(err); + } + } + if (vm) q->vm = xe_vm_get(vm); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index abdf4a57e6e2..ba443a497b38 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -134,6 +134,19 @@ struct xe_exec_queue { struct list_head link; } lr; +#define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT 0 +#define XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT 1 +#define XE_EXEC_QUEUE_TLB_INVAL_COUNT (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) + + /** @tlb_inval: TLB invalidations exec queue state */ + struct { + /** + * @tlb_inval.dep_scheduler: The TLB invalidation + * dependency scheduler + */ + struct xe_dep_scheduler *dep_scheduler; + } tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_COUNT]; + /** @pxp: PXP info tracking */ struct { /** @pxp.type: PXP session type used by this queue */ From dba89840a920ffcab7a722299510f89d7800d79b Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:14 -0700 Subject: [PATCH 031/169] drm/xe: Add GT TLB invalidation jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GT TLB invalidation jobs which issue GT TLB invalidations. Built on top of Xe generic dependency scheduler. v2: - Fix checkpatch v3: - Fix kernel doc in xe_gt_tlb_inval_job_alloc_dep, xe_gt_tlb_inval_job_push - Use IS_ERR_OR_NULL in xe_gt_tlb_inval_job_put - Squash migrate lock / unlock helpers into this patch (Stuart) Suggested-by: Thomas Hellström Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-6-matthew.brost@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c | 274 +++++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h | 34 +++ drivers/gpu/drm/xe/xe_migrate.c | 36 +++ drivers/gpu/drm/xe/xe_migrate.h | 4 + 5 files changed, 349 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c create mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 6e1fd6560557..7eaf4054c99e 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -62,6 +62,7 @@ xe-y += xe_bb.o \ xe_gt_sysfs.o \ xe_gt_throttle.o \ xe_gt_tlb_invalidation.o \ + xe_gt_tlb_inval_job.o \ xe_gt_topology.o \ xe_guc.o \ xe_guc_ads.o \ diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c new file mode 100644 index 000000000000..e9255be26467 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_dep_job_types.h" +#include "xe_dep_scheduler.h" +#include "xe_exec_queue.h" +#include "xe_gt.h" +#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval_job.h" +#include "xe_migrate.h" +#include "xe_pm.h" + +/** struct xe_gt_tlb_inval_job - GT TLB invalidation job */ +struct xe_gt_tlb_inval_job { + /** @dep: base generic dependency Xe job */ + struct xe_dep_job dep; + /** @gt: GT to invalidate */ + struct xe_gt *gt; + /** @q: exec queue issuing the invalidate */ + struct xe_exec_queue *q; + /** @refcount: ref count of this job */ + struct kref refcount; + /** + * @fence: dma fence to indicate completion. 1 way relationship - job + * can safely reference fence, fence cannot safely reference job. + */ + struct dma_fence *fence; + /** @start: Start address to invalidate */ + u64 start; + /** @end: End address to invalidate */ + u64 end; + /** @asid: Address space ID to invalidate */ + u32 asid; + /** @fence_armed: Fence has been armed */ + bool fence_armed; +}; + +static struct dma_fence *xe_gt_tlb_inval_job_run(struct xe_dep_job *dep_job) +{ + struct xe_gt_tlb_inval_job *job = + container_of(dep_job, typeof(*job), dep); + struct xe_gt_tlb_invalidation_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + + xe_gt_tlb_invalidation_range(job->gt, ifence, job->start, + job->end, job->asid); + + return job->fence; +} + +static void xe_gt_tlb_inval_job_free(struct xe_dep_job *dep_job) +{ + struct xe_gt_tlb_inval_job *job = + container_of(dep_job, typeof(*job), dep); + + /* Pairs with get in xe_gt_tlb_inval_job_push */ + xe_gt_tlb_inval_job_put(job); +} + +static const struct xe_dep_job_ops dep_job_ops = { + .run_job = xe_gt_tlb_inval_job_run, + .free_job = xe_gt_tlb_inval_job_free, +}; + +static int xe_gt_tlb_inval_context(struct xe_gt *gt) +{ + return xe_gt_is_media_type(gt) ? XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT : + XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; +} + +/** + * xe_gt_tlb_inval_job_create() - GT TLB invalidation job create + * @gt: GT to invalidate + * @q: exec queue issuing the invalidate + * @start: Start address to invalidate + * @end: End address to invalidate + * @asid: Address space ID to invalidate + * + * Create a GT TLB invalidation job and initialize internal fields. The caller is + * responsible for releasing the creation reference. + * + * Return: GT TLB invalidation job object on success, ERR_PTR failure + */ +struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, + struct xe_gt *gt, + u64 start, u64 end, + u32 asid) +{ + struct xe_gt_tlb_inval_job *job; + struct xe_dep_scheduler *dep_scheduler = + q->tlb_inval[xe_gt_tlb_inval_context(gt)].dep_scheduler; + struct drm_sched_entity *entity = + xe_dep_scheduler_entity(dep_scheduler); + struct xe_gt_tlb_invalidation_fence *ifence; + int err; + + job = kmalloc(sizeof(*job), GFP_KERNEL); + if (!job) + return ERR_PTR(-ENOMEM); + + job->q = q; + job->gt = gt; + job->start = start; + job->end = end; + job->asid = asid; + job->fence_armed = false; + job->dep.ops = &dep_job_ops; + kref_init(&job->refcount); + xe_exec_queue_get(q); /* Pairs with put in xe_gt_tlb_inval_job_destroy */ + + ifence = kmalloc(sizeof(*ifence), GFP_KERNEL); + if (!ifence) { + err = -ENOMEM; + goto err_job; + } + job->fence = &ifence->base; + + err = drm_sched_job_init(&job->dep.drm, entity, 1, NULL, + q->xef ? q->xef->drm->client_id : 0); + if (err) + goto err_fence; + + /* Pairs with put in xe_gt_tlb_inval_job_destroy */ + xe_pm_runtime_get_noresume(gt_to_xe(job->gt)); + + return job; + +err_fence: + kfree(ifence); +err_job: + xe_exec_queue_put(q); + kfree(job); + + return ERR_PTR(err); +} + +static void xe_gt_tlb_inval_job_destroy(struct kref *ref) +{ + struct xe_gt_tlb_inval_job *job = container_of(ref, typeof(*job), + refcount); + struct xe_gt_tlb_invalidation_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + struct xe_device *xe = gt_to_xe(job->gt); + struct xe_exec_queue *q = job->q; + + if (!job->fence_armed) + kfree(ifence); + else + /* Ref from xe_gt_tlb_invalidation_fence_init */ + dma_fence_put(job->fence); + + drm_sched_job_cleanup(&job->dep.drm); + kfree(job); + xe_exec_queue_put(q); /* Pairs with get from xe_gt_tlb_inval_job_create */ + xe_pm_runtime_put(xe); /* Pairs with get from xe_gt_tlb_inval_job_create */ +} + +/** + * xe_gt_tlb_inval_alloc_dep() - GT TLB invalidation job alloc dependency + * @job: GT TLB invalidation job to alloc dependency for + * + * Allocate storage for a dependency in the GT TLB invalidation fence. This + * function should be called at most once per job and must be paired with + * xe_gt_tlb_inval_job_push being called with a real fence. + * + * Return: 0 on success, -errno on failure + */ +int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job) +{ + xe_assert(gt_to_xe(job->gt), !xa_load(&job->dep.drm.dependencies, 0)); + might_alloc(GFP_KERNEL); + + return drm_sched_job_add_dependency(&job->dep.drm, + dma_fence_get_stub()); +} + +/** + * xe_gt_tlb_inval_job_push() - GT TLB invalidation job push + * @job: GT TLB invalidation job to push + * @m: The migration object being used + * @fence: Dependency for GT TLB invalidation job + * + * Pushes a GT TLB invalidation job for execution, using @fence as a dependency. + * Storage for @fence must be preallocated with xe_gt_tlb_inval_job_alloc_dep + * prior to this call if @fence is not signaled. Takes a reference to the job’s + * finished fence, which the caller is responsible for releasing, and return it + * to the caller. This function is safe to be called in the path of reclaim. + * + * Return: Job's finished fence on success, cannot fail + */ +struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence) +{ + struct xe_gt_tlb_invalidation_fence *ifence = + container_of(job->fence, typeof(*ifence), base); + + if (!dma_fence_is_signaled(fence)) { + void *ptr; + + /* + * Can be in path of reclaim, hence the preallocation of fence + * storage in xe_gt_tlb_inval_job_alloc_dep. Verify caller did + * this correctly. + */ + xe_assert(gt_to_xe(job->gt), + xa_load(&job->dep.drm.dependencies, 0) == + dma_fence_get_stub()); + + dma_fence_get(fence); /* ref released once dependency processed by scheduler */ + ptr = xa_store(&job->dep.drm.dependencies, 0, fence, + GFP_ATOMIC); + xe_assert(gt_to_xe(job->gt), !xa_is_err(ptr)); + } + + xe_gt_tlb_inval_job_get(job); /* Pairs with put in free_job */ + job->fence_armed = true; + + /* + * We need the migration lock to protect the job's seqno and the spsc + * queue, only taken on migration queue, user queues protected dma-resv + * VM lock. + */ + xe_migrate_job_lock(m, job->q); + + /* Creation ref pairs with put in xe_gt_tlb_inval_job_destroy */ + xe_gt_tlb_invalidation_fence_init(job->gt, ifence, false); + dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */ + + drm_sched_job_arm(&job->dep.drm); + /* + * caller ref, get must be done before job push as it could immediately + * signal and free. + */ + dma_fence_get(&job->dep.drm.s_fence->finished); + drm_sched_entity_push_job(&job->dep.drm); + + xe_migrate_job_unlock(m, job->q); + + /* + * Not using job->fence, as it has its own dma-fence context, which does + * not allow GT TLB invalidation fences on the same queue, GT tuple to + * be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler + * context and job's finished fence, which enables squashing. + */ + return &job->dep.drm.s_fence->finished; +} + +/** + * xe_gt_tlb_inval_job_get() - Get a reference to GT TLB invalidation job + * @job: GT TLB invalidation job object + * + * Increment the GT TLB invalidation job's reference count + */ +void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job) +{ + kref_get(&job->refcount); +} + +/** + * xe_gt_tlb_inval_job_put() - Put a reference to GT TLB invalidation job + * @job: GT TLB invalidation job object + * + * Decrement the GT TLB invalidation job's reference count, call + * xe_gt_tlb_inval_job_destroy when reference count == 0. Skips decrement if + * input @job is NULL or IS_ERR. + */ +void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job) +{ + if (!IS_ERR_OR_NULL(job)) + kref_put(&job->refcount, xe_gt_tlb_inval_job_destroy); +} diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h new file mode 100644 index 000000000000..883896194a34 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GT_TLB_INVAL_JOB_H_ +#define _XE_GT_TLB_INVAL_JOB_H_ + +#include + +struct dma_fence; +struct drm_sched_job; +struct kref; +struct xe_exec_queue; +struct xe_gt; +struct xe_gt_tlb_inval_job; +struct xe_migrate; + +struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, + struct xe_gt *gt, + u64 start, u64 end, + u32 asid); + +int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job); + +struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence); + +void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job); + +void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job); + +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index ed04c2b368dc..90065d7d29ff 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -2077,6 +2077,42 @@ out_err: return IS_ERR(fence) ? PTR_ERR(fence) : 0; } +/** + * xe_migrate_job_lock() - Lock migrate job lock + * @m: The migration context. + * @q: Queue associated with the operation which requires a lock + * + * Lock the migrate job lock if the queue is a migration queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q) +{ + bool is_migrate = q == m->q; + + if (is_migrate) + mutex_lock(&m->job_mutex); + else + xe_vm_assert_held(q->vm); /* User queues VM's should be locked */ +} + +/** + * xe_migrate_job_unlock() - Unlock migrate job lock + * @m: The migration context. + * @q: Queue associated with the operation which requires a lock + * + * Unlock the migrate job lock if the queue is a migration queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q) +{ + bool is_migrate = q == m->q; + + if (is_migrate) + mutex_unlock(&m->job_mutex); + else + xe_vm_assert_held(q->vm); /* User queues VM's should be locked */ +} + #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) #include "tests/xe_migrate.c" #endif diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index f4dac270dba3..5af2f5aa645f 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -141,4 +141,8 @@ xe_migrate_update_pgtables(struct xe_migrate *m, struct xe_migrate_pt_update *pt_update); void xe_migrate_wait(struct xe_migrate *m); + +void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q); +void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q); + #endif From b8d5779eee382003b26b1a147071adf08c5d294c Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:15 -0700 Subject: [PATCH 032/169] drm/xe: Use GT TLB invalidation jobs in PT layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than open-coding GT TLB invalidations in the PT layer, use GT TLB invalidation jobs. The real benefit is that GT TLB invalidation jobs use a single dma-fence context, allowing the generated fences to be squashed in dma-resv/DRM scheduler. v2: - s/;;/; (checkpatch) - Move ijob/mjob job push after range fence install v3: - Remove extra newline (Stuart) - Set ijob/mjob near creation (Stuart) - Add comment back in (Stuart) Suggested-by: Thomas Hellström Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-7-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_migrate.h | 9 ++ drivers/gpu/drm/xe/xe_pt.c | 178 +++++++++++++------------------- 2 files changed, 80 insertions(+), 107 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 5af2f5aa645f..3758f9615484 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -14,6 +14,7 @@ struct ttm_resource; struct xe_bo; struct xe_gt; +struct xe_gt_tlb_inval_job; struct xe_exec_queue; struct xe_migrate; struct xe_migrate_pt_update; @@ -91,6 +92,14 @@ struct xe_migrate_pt_update { struct xe_vma_ops *vops; /** @job: The job if a GPU page-table update. NULL otherwise */ struct xe_sched_job *job; + /** + * @ijob: The GT TLB invalidation job for primary tile. NULL otherwise + */ + struct xe_gt_tlb_inval_job *ijob; + /** + * @mjob: The GT TLB invalidation job for media tile. NULL otherwise + */ + struct xe_gt_tlb_inval_job *mjob; /** @tile_id: Tile ID of the update */ u8 tile_id; }; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index c8e63bd23300..330cc0f54a3f 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -13,7 +13,7 @@ #include "xe_drm_client.h" #include "xe_exec_queue.h" #include "xe_gt.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval_job.h" #include "xe_migrate.h" #include "xe_pt_types.h" #include "xe_pt_walk.h" @@ -1261,6 +1261,8 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, } static int xe_pt_vm_dependencies(struct xe_sched_job *job, + struct xe_gt_tlb_inval_job *ijob, + struct xe_gt_tlb_inval_job *mjob, struct xe_vm *vm, struct xe_vma_ops *vops, struct xe_vm_pgtable_update_ops *pt_update_ops, @@ -1328,6 +1330,20 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, for (i = 0; job && !err && i < vops->num_syncs; i++) err = xe_sync_entry_add_deps(&vops->syncs[i], job); + if (job) { + if (ijob) { + err = xe_gt_tlb_inval_job_alloc_dep(ijob); + if (err) + return err; + } + + if (mjob) { + err = xe_gt_tlb_inval_job_alloc_dep(mjob); + if (err) + return err; + } + } + return err; } @@ -1339,7 +1355,8 @@ static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[pt_update->tile_id]; - return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops, + return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob, + pt_update->mjob, vm, pt_update->vops, pt_update_ops, rftree); } @@ -1509,75 +1526,6 @@ static int xe_pt_svm_pre_commit(struct xe_migrate_pt_update *pt_update) } #endif -struct invalidation_fence { - struct xe_gt_tlb_invalidation_fence base; - struct xe_gt *gt; - struct dma_fence *fence; - struct dma_fence_cb cb; - struct work_struct work; - u64 start; - u64 end; - u32 asid; -}; - -static void invalidation_fence_cb(struct dma_fence *fence, - struct dma_fence_cb *cb) -{ - struct invalidation_fence *ifence = - container_of(cb, struct invalidation_fence, cb); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_cb(xe, &ifence->base); - if (!ifence->fence->error) { - queue_work(system_wq, &ifence->work); - } else { - ifence->base.base.error = ifence->fence->error; - xe_gt_tlb_invalidation_fence_signal(&ifence->base); - } - dma_fence_put(ifence->fence); -} - -static void invalidation_fence_work_func(struct work_struct *w) -{ - struct invalidation_fence *ifence = - container_of(w, struct invalidation_fence, work); - struct xe_device *xe = gt_to_xe(ifence->gt); - - trace_xe_gt_tlb_invalidation_fence_work_func(xe, &ifence->base); - xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start, - ifence->end, ifence->asid); -} - -static void invalidation_fence_init(struct xe_gt *gt, - struct invalidation_fence *ifence, - struct dma_fence *fence, - u64 start, u64 end, u32 asid) -{ - int ret; - - trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base); - - xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false); - - ifence->fence = fence; - ifence->gt = gt; - ifence->start = start; - ifence->end = end; - ifence->asid = asid; - - INIT_WORK(&ifence->work, invalidation_fence_work_func); - ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb); - if (ret == -ENOENT) { - dma_fence_put(ifence->fence); /* Usually dropped in CB */ - invalidation_fence_work_func(&ifence->work); - } else if (ret) { - dma_fence_put(&ifence->base.base); /* Caller ref */ - dma_fence_put(&ifence->base.base); /* Creation ref */ - } - - xe_gt_assert(gt, !ret || ret == -ENOENT); -} - struct xe_pt_stage_unbind_walk { /** @base: The pagewalk base-class. */ struct xe_pt_walk base; @@ -2407,8 +2355,8 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) struct xe_vm *vm = vops->vm; struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id]; - struct dma_fence *fence; - struct invalidation_fence *ifence = NULL, *mfence = NULL; + struct dma_fence *fence, *ifence, *mfence; + struct xe_gt_tlb_inval_job *ijob = NULL, *mjob = NULL; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; struct xe_range_fence *rfence; @@ -2440,26 +2388,38 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) #endif if (pt_update_ops->needs_invalidation) { - ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!ifence) { - err = -ENOMEM; + ijob = xe_gt_tlb_inval_job_create(pt_update_ops->q, + tile->primary_gt, + pt_update_ops->start, + pt_update_ops->last, + vm->usm.asid); + if (IS_ERR(ijob)) { + err = PTR_ERR(ijob); goto kill_vm_tile1; } + update.ijob = ijob; + if (tile->media_gt) { - mfence = kzalloc(sizeof(*ifence), GFP_KERNEL); - if (!mfence) { - err = -ENOMEM; - goto free_ifence; + mjob = xe_gt_tlb_inval_job_create(pt_update_ops->q, + tile->media_gt, + pt_update_ops->start, + pt_update_ops->last, + vm->usm.asid); + if (IS_ERR(mjob)) { + err = PTR_ERR(mjob); + goto free_ijob; } + update.mjob = mjob; + fences = kmalloc_array(2, sizeof(*fences), GFP_KERNEL); if (!fences) { err = -ENOMEM; - goto free_ifence; + goto free_ijob; } cf = dma_fence_array_alloc(2); if (!cf) { err = -ENOMEM; - goto free_ifence; + goto free_ijob; } } } @@ -2467,7 +2427,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) rfence = kzalloc(sizeof(*rfence), GFP_KERNEL); if (!rfence) { err = -ENOMEM; - goto free_ifence; + goto free_ijob; } fence = xe_migrate_update_pgtables(tile->migrate, &update); @@ -2491,30 +2451,31 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) pt_update_ops->last, fence)) dma_fence_wait(fence, false); - /* tlb invalidation must be done before signaling rebind */ - if (ifence) { - if (mfence) - dma_fence_get(fence); - invalidation_fence_init(tile->primary_gt, ifence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - if (mfence) { - invalidation_fence_init(tile->media_gt, mfence, fence, - pt_update_ops->start, - pt_update_ops->last, vm->usm.asid); - fences[0] = &ifence->base.base; - fences[1] = &mfence->base.base; + /* tlb invalidation must be done before signaling unbind/rebind */ + if (ijob) { + struct dma_fence *__fence; + + ifence = xe_gt_tlb_inval_job_push(ijob, tile->migrate, fence); + __fence = ifence; + + if (mjob) { + fences[0] = ifence; + mfence = xe_gt_tlb_inval_job_push(mjob, tile->migrate, + fence); + fences[1] = mfence; + dma_fence_array_init(cf, 2, fences, vm->composite_fence_ctx, vm->composite_fence_seqno++, false); - fence = &cf->base; - } else { - fence = &ifence->base.base; + __fence = &cf->base; } + + dma_fence_put(fence); + fence = __fence; } - if (!mfence) { + if (!mjob) { dma_resv_add_fence(xe_vm_resv(vm), fence, pt_update_ops->wait_vm_bookkeep ? DMA_RESV_USAGE_KERNEL : @@ -2523,19 +2484,19 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) list_for_each_entry(op, &vops->list, link) op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); } else { - dma_resv_add_fence(xe_vm_resv(vm), &ifence->base.base, + dma_resv_add_fence(xe_vm_resv(vm), ifence, pt_update_ops->wait_vm_bookkeep ? DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_BOOKKEEP); - dma_resv_add_fence(xe_vm_resv(vm), &mfence->base.base, + dma_resv_add_fence(xe_vm_resv(vm), mfence, pt_update_ops->wait_vm_bookkeep ? DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_BOOKKEEP); list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, - &ifence->base.base, &mfence->base.base); + op_commit(vops->vm, tile, pt_update_ops, op, ifence, + mfence); } if (pt_update_ops->needs_svm_lock) @@ -2543,15 +2504,18 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) if (pt_update_ops->needs_userptr_lock) up_read(&vm->userptr.notifier_lock); + xe_gt_tlb_inval_job_put(mjob); + xe_gt_tlb_inval_job_put(ijob); + return fence; free_rfence: kfree(rfence); -free_ifence: +free_ijob: kfree(cf); kfree(fences); - kfree(mfence); - kfree(ifence); + xe_gt_tlb_inval_job_put(mjob); + xe_gt_tlb_inval_job_put(ijob); kill_vm_tile1: if (err != -EAGAIN && err != -ENODATA && tile->id) xe_vm_kill(vops->vm, false); From 51330ba66caa29c1a73408fada7f824b36b3238f Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 24 Jul 2025 12:12:16 -0700 Subject: [PATCH 033/169] drm/xe: Remove unused GT TLB invalidation trace points Remove unused GT TLB invalidation trace points after converting to used GT TLB invalidation jobs. The trace points removed were used during early bring up of unstable driver, with a stable driver no need to replace with new tracepoints. Signed-off-by: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250724191216.4076566-8-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_trace.h | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index b4a3577df70c..21486a6f693a 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -45,22 +45,6 @@ DECLARE_EVENT_CLASS(xe_gt_tlb_invalidation_fence, __get_str(dev), __entry->fence, __entry->seqno) ); -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_create, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), - TP_ARGS(xe, fence) -); - -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, - xe_gt_tlb_invalidation_fence_work_func, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), - TP_ARGS(xe, fence) -); - -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_cb, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), - TP_ARGS(xe, fence) -); - DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_send, TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), TP_ARGS(xe, fence) From f98de826b418885a21ece67f0f5b921ae759b7bf Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Thu, 24 Jul 2025 19:38:55 +0000 Subject: [PATCH 034/169] drm/xe/hw_engine_group: Avoid call kfree() for drmm_kzalloc() Memory allocated with drmm_kzalloc() should not be freed using kfree(), as it is managed by the DRM subsystem. The memory will be automatically freed when the associated drm_device is released. These 3 group pointers are allocated using drmm_kzalloc() in hw_engine_group_alloc(), so they don't require manual deallocation. Fixes: 67979060740f ("drm/xe/hw_engine_group: Fix potential leak") Cc: Michal Wajdeczko Cc: Matthew Brost Signed-off-by: Shuicheng Lin Reviewed-by: Matthew Brost Signed-off-by: Michal Wajdeczko Link: https://lore.kernel.org/r/20250724193854.1124510-2-shuicheng.lin@intel.com --- drivers/gpu/drm/xe/xe_hw_engine_group.c | 28 ++++++------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index 87a6dcb1b4b5..c926f840c87b 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -75,25 +75,18 @@ int xe_hw_engine_setup_groups(struct xe_gt *gt) enum xe_hw_engine_id id; struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; struct xe_device *xe = gt_to_xe(gt); - int err; group_rcs_ccs = hw_engine_group_alloc(xe); - if (IS_ERR(group_rcs_ccs)) { - err = PTR_ERR(group_rcs_ccs); - goto err_group_rcs_ccs; - } + if (IS_ERR(group_rcs_ccs)) + return PTR_ERR(group_rcs_ccs); group_bcs = hw_engine_group_alloc(xe); - if (IS_ERR(group_bcs)) { - err = PTR_ERR(group_bcs); - goto err_group_bcs; - } + if (IS_ERR(group_bcs)) + return PTR_ERR(group_bcs); group_vcs_vecs = hw_engine_group_alloc(xe); - if (IS_ERR(group_vcs_vecs)) { - err = PTR_ERR(group_vcs_vecs); - goto err_group_vcs_vecs; - } + if (IS_ERR(group_vcs_vecs)) + return PTR_ERR(group_vcs_vecs); for_each_hw_engine(hwe, gt, id) { switch (hwe->class) { @@ -116,15 +109,6 @@ int xe_hw_engine_setup_groups(struct xe_gt *gt) } return 0; - -err_group_vcs_vecs: - kfree(group_vcs_vecs); -err_group_bcs: - kfree(group_bcs); -err_group_rcs_ccs: - kfree(group_rcs_ccs); - - return err; } /** From 0bdd05c2a82bbf2419415d012fd4f5faeca7f1af Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 16:10:54 +0200 Subject: [PATCH 035/169] drm/xe/configfs: Fix pci_dev reference leak We are using pci_get_domain_bus_and_slot() function to verify if the given config directory name matches any existing PCI device, but we missed to call matching pci_dev_put() to release reference. While around, also change error code in case of no device match, to make it more specific than generic formatting error. Fixes: 16280ded45fb ("drm/xe: Add configfs to enable survivability mode") Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Reviewed-by: Lucas De Marchi Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250722141059.30707-2-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 8ec1ff1e4e80..e9b46a2d0019 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -267,7 +267,8 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); if (!pdev) - return ERR_PTR(-EINVAL); + return ERR_PTR(-ENODEV); + pci_dev_put(pdev); dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) From 400a6da1e967c4f117e4757412df06dcfaea0e6a Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 16:10:55 +0200 Subject: [PATCH 036/169] drm/xe/configfs: Enforce canonical device names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While we expect config directory names to match PCI device name, currently we are only scanning provided names for domain, bus, device and function numbers, without checking their format. This would pass slightly broken entries like: /sys/kernel/config/xe/ ├── 0000:00:02.0000000000000 │   └── ... ├── 0000:00:02.0x │   └── ... ├── 0: 0: 2. 0 │   └── ... └── 0:0:2.0 └── ... To avoid such mistakes, check if the name provided exactly matches the canonical PCI device address format, which we recreated from the parsed BDF data. Also simplify scanf format as it can't really catch all formatting errors. Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250722141059.30707-3-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index e9b46a2d0019..f5ce48a0adc8 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -259,12 +259,19 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro unsigned int domain, bus, slot, function; struct xe_config_device *dev; struct pci_dev *pdev; + char canonical[16]; int ret; - ret = sscanf(name, "%04x:%02x:%02x.%x", &domain, &bus, &slot, &function); + ret = sscanf(name, "%x:%x:%x.%x", &domain, &bus, &slot, &function); if (ret != 4) return ERR_PTR(-EINVAL); + ret = scnprintf(canonical, sizeof(canonical), "%04x:%02x:%02x.%d", domain, bus, + PCI_SLOT(PCI_DEVFN(slot, function)), + PCI_FUNC(PCI_DEVFN(slot, function))); + if (ret != 12 || strcmp(name, canonical)) + return ERR_PTR(-EINVAL); + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); if (!pdev) return ERR_PTR(-ENODEV); From 9b807f0bb0d0f4b029e56dd2daa4a5e6d87fab61 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 22 Jul 2025 16:10:56 +0200 Subject: [PATCH 037/169] drm/xe/configfs: Use pci_name() for lookup There is no need to manually build PCI device name from BDF data, since it was already prepared and assigned and can be accessed by calling pci_name() function. Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Reviewed-by: Lucas De Marchi Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250722141059.30707-4-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index f5ce48a0adc8..36e2b45b305f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -312,13 +312,9 @@ static struct configfs_subsystem xe_configfs = { static struct xe_config_device *configfs_find_group(struct pci_dev *pdev) { struct config_item *item; - char name[64]; - - snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pci_domain_nr(pdev->bus), - pdev->bus->number, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); mutex_lock(&xe_configfs.su_mutex); - item = config_group_find_item(&xe_configfs.su_group, name); + item = config_group_find_item(&xe_configfs.su_group, pci_name(pdev)); mutex_unlock(&xe_configfs.su_mutex); if (!item) From e8372edec948318934c71542e939d1a8fb8fabcf Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Fri, 11 Jul 2025 17:01:52 +0100 Subject: [PATCH 038/169] drm/xe/xelp: Implement Wa_16010904313 Add XeLP workaround 16010904313. The description calls for it to be emitted as the indirect context buffer workaround for render and compute, and from the workaround batch buffer for the other engines. Therefore we plug into the previously added respective top level emission functions. The actual command streamer programming sequence differs from what is described in the PRM, in that it assumes the listed LRCA offset was supposed to actually refer to the location of the CTX_TIMESTAMP register instead of LRCA + 0x180c (which is in GPR space). Latter appears to make more sense under the assumption that multiple writes are helping with restoring the CTX_TIMESTAMP register content from the saved context state. Signed-off-by: Tvrtko Ursulin Reviewed-by: Lucas De Marchi Cc: Matt Roper Link: https://lore.kernel.org/r/20250711160153.49833-8-tvrtko.ursulin@igalia.com Signed-off-by: Lucas De Marchi --- .../gpu/drm/xe/instructions/xe_mi_commands.h | 1 + drivers/gpu/drm/xe/xe_lrc.c | 44 +++++++++++++++++++ drivers/gpu/drm/xe/xe_wa_oob.rules | 1 + 3 files changed, 46 insertions(+) diff --git a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h index e3f5e8bb3ebc..c47b290e0e9f 100644 --- a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h @@ -65,6 +65,7 @@ #define MI_LOAD_REGISTER_MEM (__MI_INSTR(0x29) | XE_INSTR_NUM_DW(4)) #define MI_LRM_USE_GGTT REG_BIT(22) +#define MI_LRM_ASYNC REG_BIT(21) #define MI_LOAD_REGISTER_REG (__MI_INSTR(0x2a) | XE_INSTR_NUM_DW(3)) #define MI_LRR_DST_CS_MMIO REG_BIT(19) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 6d38411bdeba..110731a1e547 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -76,6 +76,11 @@ lrc_to_xe(struct xe_lrc *lrc) static bool gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) { + if (XE_WA(gt, 16010904313) && + (class == XE_ENGINE_CLASS_RENDER || + class == XE_ENGINE_CLASS_COMPUTE)) + return true; + return false; } @@ -1014,6 +1019,43 @@ static ssize_t setup_utilization_wa(struct xe_lrc *lrc, return cmd - batch; } +static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); + u32 *cmd = batch; + + if (!XE_WA(lrc->gt, 16010904313) || + !(hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE || + hwe->class == XE_ENGINE_CLASS_COPY || + hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE || + hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)) + return 0; + + if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) + return -ENOSPC; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | + MI_LRM_ASYNC; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | + MI_LRM_ASYNC; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO; + *cmd++ = RING_CTX_TIMESTAMP(0).addr; + *cmd++ = ts_addr; + *cmd++ = 0; + + return cmd - batch; +} + struct bo_setup { ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *batch, size_t max_size); @@ -1089,6 +1131,7 @@ static void finish_bo(struct bo_setup_state *state) static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) { static const struct bo_setup funcs[] = { + { .setup = setup_timestamp_wa }, { .setup = setup_utilization_wa }, }; struct bo_setup_state state = { @@ -1121,6 +1164,7 @@ static int setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) { static struct bo_setup rcs_funcs[] = { + { .setup = setup_timestamp_wa }, }; struct bo_setup_state state = { .lrc = lrc, diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 46ad72bb1f8d..e74256944208 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -1,4 +1,5 @@ 1607983814 GRAPHICS_VERSION_RANGE(1200, 1210) +16010904313 GRAPHICS_VERSION_RANGE(1200, 1210) 22012773006 GRAPHICS_VERSION_RANGE(1200, 1250) 14014475959 GRAPHICS_VERSION_RANGE(1270, 1271), GRAPHICS_STEP(A0, B0) PLATFORM(DG2) From ca33cd271ef9b27acddf96742eef136d0f4caaa5 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Fri, 11 Jul 2025 17:01:53 +0100 Subject: [PATCH 039/169] drm/xe/xelp: Add Wa_18022495364 Add Wa_18022495364 as a context workaround batch buffer workaround. Signed-off-by: Tvrtko Ursulin Reviewed-by: Lucas De Marchi Cc: Matt Roper Link: https://lore.kernel.org/r/20250711160153.49833-9-tvrtko.ursulin@igalia.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/regs/xe_engine_regs.h | 3 +++ drivers/gpu/drm/xe/xe_lrc.c | 21 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_wa_oob.rules | 1 + 3 files changed, 25 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index 7ade41e2b7b3..f4c3e1187a00 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -111,6 +111,9 @@ #define PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS REG_BIT(14) #define CS_PRIORITY_MEM_READ REG_BIT(7) +#define CS_DEBUG_MODE2(base) XE_REG((base) + 0xd8, XE_REG_OPTION_MASKED) +#define INSTRUCTION_STATE_CACHE_INVALIDATE REG_BIT(6) + #define FF_SLICE_CS_CHICKEN1(base) XE_REG((base) + 0xe0, XE_REG_OPTION_MASKED) #define FFSC_PERCTX_PREEMPT_CTRL REG_BIT(14) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 110731a1e547..540f044bf425 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1056,6 +1056,26 @@ static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, return cmd - batch; } +static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, + struct xe_hw_engine *hwe, + u32 *batch, size_t max_len) +{ + u32 *cmd = batch; + + if (!XE_WA(lrc->gt, 18022495364) || + hwe->class != XE_ENGINE_CLASS_RENDER) + return 0; + + if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) + return -ENOSPC; + + *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cmd++ = CS_DEBUG_MODE1(0).addr; + *cmd++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); + + return cmd - batch; +} + struct bo_setup { ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *batch, size_t max_size); @@ -1132,6 +1152,7 @@ static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) { static const struct bo_setup funcs[] = { { .setup = setup_timestamp_wa }, + { .setup = setup_invalidate_state_cache_wa }, { .setup = setup_utilization_wa }, }; struct bo_setup_state state = { diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index e74256944208..8d0aabab6777 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -1,5 +1,6 @@ 1607983814 GRAPHICS_VERSION_RANGE(1200, 1210) 16010904313 GRAPHICS_VERSION_RANGE(1200, 1210) +18022495364 GRAPHICS_VERSION_RANGE(1200, 1210) 22012773006 GRAPHICS_VERSION_RANGE(1200, 1250) 14014475959 GRAPHICS_VERSION_RANGE(1270, 1271), GRAPHICS_STEP(A0, B0) PLATFORM(DG2) From 1ffcf8b8ae8a786b61350ee082b6c82846a31e53 Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Mon, 14 Jul 2025 15:26:58 +0300 Subject: [PATCH 040/169] drm/xe: Support for mmap-ing mmio regions Allow the driver to expose hardware register spaces to userspace through GEM objects with fake mmap offsets. This can be useful for userspace-firmware communication, debugging, etc. v2: Minor doc fix (CI) v3: Enforce MAP_SHARED (Tejas) Add fault handler with dummy page (Tejas, Matt Auld) Store physical address instead of xe_mmio in the GEM object (MattB) v4: Separate xe_mmio_gem from xe_mmio and make it private (MattB) Signed-off-by: Ilia Levi Reviewed-by: Matthew Brost Reviewed-by: Tejas Upadhyay Link: https://lore.kernel.org/r/20250714122658.1803-1-ilia.levi@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_mmio_gem.c | 226 +++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_mmio_gem.h | 20 +++ 3 files changed, 247 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_mmio_gem.c create mode 100644 drivers/gpu/drm/xe/xe_mmio_gem.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 7eaf4054c99e..42c6ca5b3f76 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -87,6 +87,7 @@ xe-y += xe_bb.o \ xe_lrc.o \ xe_migrate.o \ xe_mmio.o \ + xe_mmio_gem.o \ xe_mocs.o \ xe_module.o \ xe_nvm.o \ diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c new file mode 100644 index 000000000000..9a97c4387e4f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_mmio_gem.h" + +#include +#include +#include + +#include "xe_device_types.h" + +/** + * DOC: Exposing MMIO regions to userspace + * + * In certain cases, the driver may allow userspace to mmap a portion of the hardware registers. + * + * This can be done as follows: + * 1. Call xe_mmio_gem_create() to create a GEM object with an mmap-able fake offset. + * 2. Use xe_mmio_gem_mmap_offset() on the created GEM object to retrieve the fake offset. + * 3. Provide the fake offset to userspace. + * 4. Userspace can call mmap with the fake offset. The length provided to mmap + * must match the size of the GEM object. + * 5. When the region is no longer needed, call xe_mmio_gem_destroy() to release the GEM object. + * + * NOTE: The exposed MMIO region must be page-aligned with regards to its BAR offset and size. + * + * WARNING: Exposing MMIO regions to userspace can have security and stability implications. + * Make sure not to expose any sensitive registers. + */ + +static void xe_mmio_gem_free(struct drm_gem_object *); +static int xe_mmio_gem_mmap(struct drm_gem_object *, struct vm_area_struct *); +static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *); + +struct xe_mmio_gem { + struct drm_gem_object base; + phys_addr_t phys_addr; +}; + +static const struct vm_operations_struct vm_ops = { + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, + .fault = xe_mmio_gem_vm_fault, +}; + +static const struct drm_gem_object_funcs xe_mmio_gem_funcs = { + .free = xe_mmio_gem_free, + .mmap = xe_mmio_gem_mmap, + .vm_ops = &vm_ops, +}; + +static inline struct xe_mmio_gem *to_xe_mmio_gem(struct drm_gem_object *obj) +{ + return container_of(obj, struct xe_mmio_gem, base); +} + +/** + * xe_mmio_gem_create - Expose an MMIO region to userspace + * @xe: The xe device + * @file: DRM file descriptor + * @phys_addr: Start of the exposed MMIO region + * @size: The size of the exposed MMIO region + * + * This function creates a GEM object that exposes an MMIO region with an mmap-able + * fake offset. + * + * See: "Exposing MMIO regions to userspace" + */ +struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, + phys_addr_t phys_addr, size_t size) +{ + struct xe_mmio_gem *obj; + struct drm_gem_object *base; + int err; + + if ((phys_addr % PAGE_SIZE != 0) || (size % PAGE_SIZE != 0)) + return ERR_PTR(-EINVAL); + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) + return ERR_PTR(-ENOMEM); + + base = &obj->base; + base->funcs = &xe_mmio_gem_funcs; + obj->phys_addr = phys_addr; + + drm_gem_private_object_init(&xe->drm, base, size); + + err = drm_gem_create_mmap_offset(base); + if (err) + goto free_gem; + + err = drm_vma_node_allow(&base->vma_node, file); + if (err) + goto free_gem; + + return obj; + +free_gem: + xe_mmio_gem_free(base); + return ERR_PTR(err); +} + +/** + * xe_mmio_gem_mmap_offset - Return the mmap-able fake offset + * @gem: the GEM object created with xe_mmio_gem_create() + * + * This function returns the mmap-able fake offset allocated during + * xe_mmio_gem_create(). + * + * See: "Exposing MMIO regions to userspace" + */ +u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem) +{ + return drm_vma_node_offset_addr(&gem->base.vma_node); +} + +static void xe_mmio_gem_free(struct drm_gem_object *base) +{ + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + + drm_gem_object_release(base); + kfree(obj); +} + +/** + * xe_mmio_gem_destroy - Destroy the GEM object that exposes an MMIO region + * @gem: the GEM object to destroy + * + * This function releases resources associated with the GEM object created by + * xe_mmio_gem_create(). + * + * See: "Exposing MMIO regions to userspace" + */ +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem) +{ + xe_mmio_gem_free(&gem->base); +} + +static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *vma) +{ + if (vma->vm_end - vma->vm_start != base->size) + return -EINVAL; + + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + + /* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */ + vma->vm_pgoff = 0; + vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | + VM_DONTCOPY | VM_NORESERVE); + + /* Defer actual mapping to the fault handler. */ + return 0; +} + +static void xe_mmio_gem_release_dummy_page(struct drm_device *dev, void *res) +{ + __free_page((struct page *)res); +} + +static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) +{ + struct drm_gem_object *base = vma->vm_private_data; + struct drm_device *dev = base->dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + struct page *page; + unsigned long pfn; + unsigned long i; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + if (drmm_add_action_or_reset(dev, xe_mmio_gem_release_dummy_page, page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Map the entire VMA to the same dummy page */ + for (i = 0; i < base->size; i += PAGE_SIZE) { + unsigned long addr = vma->vm_start + i; + + ret = vmf_insert_pfn(vma, addr, pfn); + if (ret & VM_FAULT_ERROR) + break; + } + + return ret; +} + +static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *base = vma->vm_private_data; + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + struct drm_device *dev = base->dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long i; + int idx; + + if (!drm_dev_enter(dev, &idx)) { + /* + * Provide a dummy page to avoid SIGBUS for events such as hot-unplug. + * This gives the userspace the option to recover instead of crashing. + * It is assumed the userspace will receive the notification via some + * other channel (e.g. drm uevent). + */ + return xe_mmio_gem_vm_fault_dummy_page(vma); + } + + for (i = 0; i < base->size; i += PAGE_SIZE) { + unsigned long addr = vma->vm_start + i; + unsigned long phys_addr = obj->phys_addr + i; + + ret = vmf_insert_pfn(vma, addr, PHYS_PFN(phys_addr)); + if (ret & VM_FAULT_ERROR) + break; + } + + drm_dev_exit(idx); + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.h b/drivers/gpu/drm/xe/xe_mmio_gem.h new file mode 100644 index 000000000000..4b76d5586ebb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio_gem.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_MMIO_GEM_H_ +#define _XE_MMIO_GEM_H_ + +#include + +struct drm_file; +struct xe_device; +struct xe_mmio_gem; + +struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, + phys_addr_t phys_addr, size_t size); +u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem); +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem); + +#endif /* _XE_MMIO_GEM_H_ */ From 45fbb51050e72723c2bdcedc1ce32305256c70ed Mon Sep 17 00:00:00 2001 From: John Harrison Date: Fri, 25 Jul 2025 19:43:37 -0700 Subject: [PATCH 041/169] drm/xe/guc: Add more GuC load error status codes The GuC load process will abort if certain status codes (which are indicative of a fatal error) are reported. Otherwise, it keeps waiting until the 'success' code is returned. New error codes have been added in recent GuC releases, so add support for aborting on those as well. v2: Shuffle HWCONFIG_START to the front of the switch to keep the ordering as per the enum define for clarity (review feedback by Jonathan). Also add a description for the basic 'invalid init data' code which was missing. Signed-off-by: John Harrison Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250726024337.4056272-1-John.C.Harrison@Intel.com --- drivers/gpu/drm/xe/abi/guc_errors_abi.h | 3 +++ drivers/gpu/drm/xe/xe_guc.c | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/abi/guc_errors_abi.h b/drivers/gpu/drm/xe/abi/guc_errors_abi.h index ecf748fd87df..ad76b4baf42e 100644 --- a/drivers/gpu/drm/xe/abi/guc_errors_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_errors_abi.h @@ -63,6 +63,7 @@ enum xe_guc_load_status { XE_GUC_LOAD_STATUS_HWCONFIG_START = 0x05, XE_GUC_LOAD_STATUS_HWCONFIG_DONE = 0x06, XE_GUC_LOAD_STATUS_HWCONFIG_ERROR = 0x07, + XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH = 0x08, XE_GUC_LOAD_STATUS_GDT_DONE = 0x10, XE_GUC_LOAD_STATUS_IDT_DONE = 0x20, XE_GUC_LOAD_STATUS_LAPIC_DONE = 0x30, @@ -75,6 +76,8 @@ enum xe_guc_load_status { XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_START, XE_GUC_LOAD_STATUS_MPU_DATA_INVALID = 0x73, XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID = 0x74, + XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR = 0x75, + XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG = 0x76, XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_END, XE_GUC_LOAD_STATUS_READY = 0xF0, diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index b1d1d6da3758..9e34401e4489 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -992,11 +992,14 @@ static int guc_load_done(u32 status) case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH: case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE: case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR: + case XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH: case XE_GUC_LOAD_STATUS_DPC_ERROR: case XE_GUC_LOAD_STATUS_EXCEPTION: case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID: case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID: case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: + case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR: + case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG: return -1; } @@ -1136,17 +1139,29 @@ static void guc_wait_ucode(struct xe_guc *guc) } switch (ukernel) { + case XE_GUC_LOAD_STATUS_HWCONFIG_START: + xe_gt_err(gt, "still extracting hwconfig table.\n"); + break; + case XE_GUC_LOAD_STATUS_EXCEPTION: xe_gt_err(gt, "firmware exception. EIP: %#x\n", xe_mmio_read32(mmio, SOFT_SCRATCH(13))); break; + case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID: + xe_gt_err(gt, "illegal init/ADS data\n"); + break; + case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: xe_gt_err(gt, "illegal register in save/restore workaround list\n"); break; - case XE_GUC_LOAD_STATUS_HWCONFIG_START: - xe_gt_err(gt, "still extracting hwconfig table.\n"); + case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR: + xe_gt_err(gt, "illegal workaround KLV data\n"); + break; + + case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG: + xe_gt_err(gt, "illegal feature flag specified\n"); break; } From 4df0bd5eb497c59e14924452026d6d70505706b5 Mon Sep 17 00:00:00 2001 From: Priyanka Dandamudi Date: Mon, 28 Jul 2025 10:03:36 +0530 Subject: [PATCH 042/169] drm/xe/uapi: Add documentation for DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING Add documentation for drm_xe_gem_create structure flag DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING. v2: Modified to be in a more generalised way. Signed-off-by: Priyanka Dandamudi Reviewed-by: Lucas De Marchi Link: https://patchwork.freedesktop.org/patch/msgid/20250728043336.3319521-1-priyanka.dandamudi@intel.com Signed-off-by: Tejas Upadhyay --- include/uapi/drm/xe_drm.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index e2426413488f..c721e130c1d2 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -760,7 +760,11 @@ struct drm_xe_device_query { * gem creation * * The @flags can be: - * - %DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING + * - %DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING - Modify the GEM object + * allocation strategy by deferring physical memory allocation + * until the object is either bound to a virtual memory region via + * VM_BIND or accessed by the CPU. As a result, no backing memory is + * reserved at the time of GEM object creation. * - %DRM_XE_GEM_CREATE_FLAG_SCANOUT * - %DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM - When using VRAM as a * possible placement, ensure that the corresponding VRAM allocation From d6a0311c37b0955d3b2e15137f40c65c9aeb8f20 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 25 Jul 2025 11:05:08 +0200 Subject: [PATCH 043/169] drm/xe/hw_engine_group: Don't use drm_warn to catch missed case Since hwe->class is an enumeration we can rely on the compiler to catch any unhandled engine class case at compile time thanks to [-Werror=switch]. Any unexpected use of a special CLASS_MAX enum case can be guarded by our xe_gt_assert() instead, which will be compiled-out on the production builds. Signed-off-by: Michal Wajdeczko Cc: Francois Dugast Cc: Matthew Brost Reviewed-by: Francois Dugast Link: https://lore.kernel.org/r/20250725090508.571-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_hw_engine_group.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index c926f840c87b..58bee3ffe881 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -103,8 +103,8 @@ int xe_hw_engine_setup_groups(struct xe_gt *gt) break; case XE_ENGINE_CLASS_OTHER: break; - default: - drm_warn(&xe->drm, "NOT POSSIBLE"); + case XE_ENGINE_CLASS_MAX: + xe_gt_assert(gt, false); } } From a843b9894705d5d171e97f167fc5290f6a3ecaee Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 29 Jul 2025 17:37:20 +0530 Subject: [PATCH 044/169] drm/xe/vf: Fix VM crash during VF driver release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VF CCS save/restore series (patchwork #149108) has a dependency on the migration framework. A recent migration update in commit d65ff1ec8535 ("drm/xe: Split xe_migrate allocation from initialization") caused a VM crash during XE driver release for iGPU devices. Oops: general protection fault, probably for non-canonical address 0x6b6b6b6b6b6b6b83: 0000 [#1] SMP NOPTI RIP: 0010:xe_lrc_ring_head+0x12/0xb0 [xe] Call Trace: xe_sriov_vf_ccs_fini+0x1e/0x40 [xe] devm_action_release+0x12/0x30 release_nodes+0x3a/0x120 devres_release_all+0x96/0xd0 device_unbind_cleanup+0x12/0x80 device_release_driver_internal+0x23a/0x280 device_release_driver+0x12/0x20 pci_stop_bus_device+0x69/0x90 pci_stop_and_remove_bus_device+0x12/0x30 pci_iov_remove_virtfn+0xbd/0x130 sriov_disable+0x42/0x100 pci_disable_sriov+0x34/0x50 xe_pci_sriov_configure+0xf71/0x1020 [xe] Update the VF CCS migration initialization sequence to align with the new migration framework changes, resolving the release-time crash. Fixes: f3009272ff2e ("drm/xe/vf: Create contexts for CCS read write") Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Matthew Auld Cc: Piotr Piórkowski Reviewed-by: Piotr Piórkowski Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250729120720.13990-1-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt.c | 6 ++--- drivers/gpu/drm/xe/xe_migrate.c | 37 ++++++++++++++++------------ drivers/gpu/drm/xe/xe_migrate.h | 2 +- drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 7 +++++- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index c8eda36546d3..5a79c6e3208b 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -564,11 +564,9 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) if (xe_gt_is_main_type(gt)) { struct xe_tile *tile = gt_to_tile(gt); - tile->migrate = xe_migrate_init(tile); - if (IS_ERR(tile->migrate)) { - err = PTR_ERR(tile->migrate); + err = xe_migrate_init(tile->migrate); + if (err) goto err_force_wake; - } } err = xe_uc_load_hw(>->uc); diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 90065d7d29ff..3a276e2348a2 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -396,15 +396,15 @@ struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile) /** * xe_migrate_init() - Initialize a migrate context - * @tile: Back-pointer to the tile we're initializing for. + * @m: The migration context * - * Return: Pointer to a migrate context on success. Error pointer on error. + * Return: 0 if successful, negative error code on failure */ -struct xe_migrate *xe_migrate_init(struct xe_tile *tile) +int xe_migrate_init(struct xe_migrate *m) { - struct xe_device *xe = tile_to_xe(tile); + struct xe_tile *tile = m->tile; struct xe_gt *primary_gt = tile->primary_gt; - struct xe_migrate *m = tile->migrate; + struct xe_device *xe = tile_to_xe(tile); struct xe_vm *vm; int err; @@ -412,15 +412,13 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | XE_VM_FLAG_SET_TILE_ID(tile)); if (IS_ERR(vm)) - return ERR_CAST(vm); + return PTR_ERR(vm); xe_vm_lock(vm, false); err = xe_migrate_prepare_vm(tile, m, vm); xe_vm_unlock(vm); - if (err) { - xe_vm_close_and_put(vm); - return ERR_PTR(err); - } + if (err) + goto err_out; if (xe->info.has_usm) { struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, @@ -429,8 +427,10 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) false); u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); - if (!hwe || !logical_mask) - return ERR_PTR(-EINVAL); + if (!hwe || !logical_mask) { + err = -EINVAL; + goto err_out; + } /* * XXX: Currently only reserving 1 (likely slow) BCS instance on @@ -449,8 +449,8 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) EXEC_QUEUE_FLAG_MIGRATE, 0); } if (IS_ERR(m->q)) { - xe_vm_close_and_put(vm); - return ERR_CAST(m->q); + err = PTR_ERR(m->q); + goto err_out; } mutex_init(&m->job_mutex); @@ -460,7 +460,7 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m); if (err) - return ERR_PTR(err); + return err; if (IS_DGFX(xe)) { if (xe_migrate_needs_ccs_emit(xe)) @@ -475,7 +475,12 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) (unsigned long long)m->min_chunk_size); } - return m; + return err; + +err_out: + xe_vm_close_and_put(vm); + return err; + } static u64 max_mem_transfer_per_pass(struct xe_device *xe) diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 3758f9615484..e81ea6b27fb5 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -105,7 +105,7 @@ struct xe_migrate_pt_update { }; struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile); -struct xe_migrate *xe_migrate_init(struct xe_tile *tile); +int xe_migrate_init(struct xe_migrate *m); struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, unsigned long npages, diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index af43e04179aa..bf9fa1238462 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -270,11 +270,16 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) ctx = &tile->sriov.vf.ccs[ctx_id]; ctx->ctx_id = ctx_id; - migrate = xe_migrate_init(tile); + migrate = xe_migrate_alloc(tile); if (IS_ERR(migrate)) { err = PTR_ERR(migrate); goto err_ret; } + + err = xe_migrate_init(migrate); + if (err) + goto err_ret; + ctx->migrate = migrate; err = alloc_bb_pool(tile, ctx); From 552dbba1caaf0cb40ce961806d757615e26ec668 Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Tue, 29 Jul 2025 14:34:37 +0200 Subject: [PATCH 045/169] drm/xe/vf: Disable CSC support on VF CSC is not accessible by VF drivers, so disable its support flag on VF to prevent further initialization attempts. Fixes: e02cea83d32d ("drm/xe/gsc: add Battlemage support") Signed-off-by: Lukasz Laguna Cc: Alexander Usyskin Cc: Michal Wajdeczko Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://lore.kernel.org/r/20250729123437.5933-1-lukasz.laguna@intel.com --- drivers/gpu/drm/xe/xe_device.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index d04a0ae018e6..c754374f1a8d 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -682,6 +682,7 @@ static void sriov_update_device_info(struct xe_device *xe) /* disable features that are not available/applicable to VFs */ if (IS_SRIOV_VF(xe)) { xe->info.probe_display = 0; + xe->info.has_heci_cscfi = 0; xe->info.has_heci_gscfi = 0; xe->info.skip_guc_pc = 1; xe->info.skip_pcode = 1; From 1cda3c755bb7770be07d75949bb0f45fb88651f6 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Tue, 15 Jul 2025 17:20:58 +0200 Subject: [PATCH 046/169] drm/xe: Fix oops in xe_gem_fault when running core_hotunplug test. I saw an oops in xe_gem_fault when running the xe-fast-feedback testlist against the realtime kernel without debug options enabled. The panic happens after core_hotunplug unbind-rebind finishes. Presumably what happens is that a process mmaps, unlocks because of the FAULT_FLAG_RETRY_NOWAIT logic, has no process memory left, causing ttm_bo_vm_dummy_page() to return VM_FAULT_NOPAGE, since there was nothing left to populate, and then oopses in "mem_type_is_vram(tbo->resource->mem_type)" because tbo->resource is NULL. It's convoluted, but fits the data and explains the oops after the test exits. Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20250715152057.23254-2-dev@lankhorst.se Signed-off-by: Maarten Lankhorst --- drivers/gpu/drm/xe/xe_bo.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index ffca1cea5585..1a72561f987c 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1732,22 +1732,26 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, TTM_BO_VM_NUM_PREFAULT); drm_dev_exit(idx); + + if (ret == VM_FAULT_RETRY && + !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + goto out; + + /* + * ttm_bo_vm_reserve() already has dma_resv_lock. + */ + if (ret == VM_FAULT_NOPAGE && + mem_type_is_vram(tbo->resource->mem_type)) { + mutex_lock(&xe->mem_access.vram_userfault.lock); + if (list_empty(&bo->vram_userfault_link)) + list_add(&bo->vram_userfault_link, + &xe->mem_access.vram_userfault.list); + mutex_unlock(&xe->mem_access.vram_userfault.lock); + } } else { ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); } - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - goto out; - /* - * ttm_bo_vm_reserve() already has dma_resv_lock. - */ - if (ret == VM_FAULT_NOPAGE && mem_type_is_vram(tbo->resource->mem_type)) { - mutex_lock(&xe->mem_access.vram_userfault.lock); - if (list_empty(&bo->vram_userfault_link)) - list_add(&bo->vram_userfault_link, &xe->mem_access.vram_userfault.list); - mutex_unlock(&xe->mem_access.vram_userfault.lock); - } - dma_resv_unlock(tbo->base.resv); out: if (needs_rpm) From 7d3a5962d74ee7ce2d0051d4d1e106c06b432ce8 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 1 Aug 2025 16:32:35 +0300 Subject: [PATCH 047/169] drm/xe/vf: Fix IS_ERR() vs NULL check in xe_sriov_vf_ccs_init() The xe_migrate_alloc() function returns NULL on error. It doesn't return error pointers. Update the checking to match. Fixes: a843b9894705 ("drm/xe/vf: Fix VM crash during VF driver release") Signed-off-by: Dan Carpenter Reviewed-by: Michal Wajdeczko Link: https://lore.kernel.org/r/aIzB8-Y6wtZvfNQT@stanley.mountain Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index bf9fa1238462..e363240a3455 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -271,8 +271,8 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) ctx->ctx_id = ctx_id; migrate = xe_migrate_alloc(tile); - if (IS_ERR(migrate)) { - err = PTR_ERR(migrate); + if (!migrate) { + err = -ENOMEM; goto err_ret; } From 1fdc4c381ff765479d76ccf3134717c430c871b8 Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Fri, 1 Aug 2025 10:53:56 +0530 Subject: [PATCH 048/169] drm/xe/devcoredump: Defer devcoredump initialization during probe Doing devcoredump initializing before GT though look harmless, it leads to problem during driver unbind. Because of this order, GT/Engine release functions will be called before xe devcoredump release function (xe_driver_devcoredump_fini) leading to the following kernel crash[1] because the devcoredump functions might still use GT/Engine datastructures after those are freed. The following crash is observed while running the IGT xe_wedged@wedged-at-any-timeout. The test forces a wedged state by submitting a workload which hangs. Then does a unbind/rebind of the driver to recover from the wedged state. The hanged workload leads to a devcoredump. The following crash is noticed when the devcoredump capture races with the driver unbind. During driver unbind, the release function hw_engine_fini() will be called which assigns NULL to hwe->gt. But the same data structure is accessed during the coredump capture in the function xe_engine_snapshot_print by reading snapshot->hwe->gt. With this patch, we make sure the devcoredump is stopped before deinitializing the core driver functions. [1]: BUG: kernel NULL pointer dereference, address: 0000000000000000 Workqueue: events_unbound xe_devcoredump_deferred_snap_work [xe] RIP: 0010:xe_engine_snapshot_print+0x47/0x420 [xe] Call Trace: ? drm_printf+0x64/0x90 __xe_devcoredump_read+0x23f/0x2d0 [xe] ? __pfx___drm_printfn_coredump+0x10/0x10 ? __pfx___drm_puts_coredump+0x10/0x10 xe_devcoredump_deferred_snap_work+0x17a/0x190 [xe] process_one_work+0x22e/0x6f0 worker_thread+0x1e8/0x3d0 ? __pfx_worker_thread+0x10/0x10 kthread+0x11f/0x250 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x47/0x70 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1a/0x30 v2: Detailed commit description (Rodrigo) v3: FIXME added (Rodrigo, Stuart) Fixes: 4209d635a823 ("drm/xe: Remove devcoredump during driver release") Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250731061300.14320-1-balasubramani.vivekanandan@intel.com Signed-off-by: Balasubramani Vivekanandan Link: https://lore.kernel.org/r/20250801052356.21885-1-balasubramani.vivekanandan@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 8 ++++---- drivers/gpu/drm/xe/xe_guc_capture.c | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index c754374f1a8d..57edbc63da6f 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -822,10 +822,6 @@ int xe_device_probe(struct xe_device *xe) return err; } - err = xe_devcoredump_init(xe); - if (err) - return err; - /* * From here on, if a step fails, make sure a Driver-FLR is triggereed */ @@ -890,6 +886,10 @@ int xe_device_probe(struct xe_device *xe) XE_WA(xe->tiles->media_gt, 15015404425_disable)) XE_DEVICE_WA_DISABLE(xe, 15015404425); + err = xe_devcoredump_init(xe); + if (err) + return err; + xe_nvm_init(xe); err = xe_heci_gsc_init(xe); diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 859a3ba91be5..243dad3e2418 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -1817,6 +1817,12 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm str_yes_no(snapshot->kernel_reserved)); for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { + /* + * FIXME: During devcoredump print we should avoid accessing the + * driver pointers for gt or engine. Printing should be done only + * using the snapshot captured. Here we are accessing the gt + * pointer. It should be fixed. + */ list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, capture_class, false); snapshot_print_by_list_order(snapshot, p, type, list); From b52f8d7a8f38629e3a9d1d372f2fe43e3a27d6b2 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 1 Aug 2025 16:44:18 +0200 Subject: [PATCH 049/169] drm/xe/pf: Skip LMTT update if no LMEM was provisioned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During VF unprovisioning, if VF was not provisioned with LMEM, there is no need to trigger LMTT update, as VF LMTT was never set. This will spare us sending full TLB invalidation requests. Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250801144418.180584-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index f00cce81f1ff..c8f0320d032f 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -1434,7 +1434,8 @@ fail: return err; } -static void pf_release_vf_config_lmem(struct xe_gt *gt, struct xe_gt_sriov_config *config) +/* Return: %true if there was an LMEM provisioned, %false otherwise */ +static bool pf_release_vf_config_lmem(struct xe_gt *gt, struct xe_gt_sriov_config *config) { xe_gt_assert(gt, IS_DGFX(gt_to_xe(gt))); xe_gt_assert(gt, xe_gt_is_main_type(gt)); @@ -1443,7 +1444,9 @@ static void pf_release_vf_config_lmem(struct xe_gt *gt, struct xe_gt_sriov_confi if (config->lmem_obj) { xe_bo_unpin_map_no_vm(config->lmem_obj); config->lmem_obj = NULL; + return true; } + return false; } static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) @@ -2021,12 +2024,13 @@ static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) { struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); struct xe_device *xe = gt_to_xe(gt); + bool released; if (xe_gt_is_main_type(gt)) { pf_release_vf_config_ggtt(gt, config); if (IS_DGFX(xe)) { - pf_release_vf_config_lmem(gt, config); - if (xe_device_has_lmtt(xe)) + released = pf_release_vf_config_lmem(gt, config); + if (released && xe_device_has_lmtt(xe)) pf_update_vf_lmtt(xe, vfid); } } From a424353937c24554bb242a6582ed8f018b4a411c Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 1 Aug 2025 16:28:20 +0200 Subject: [PATCH 050/169] drm/xe/pf: Disable PF restart worker on device removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can't let restart worker run once device is removed, since other data that it might want to access could be already released. Explicitly disable worker as part of device cleanup action. Fixes: a4d1c5d0b99b ("drm/xe/pf: Move VFs reprovisioning to worker") Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Cc: Jonathan Cavitt Link: https://lore.kernel.org/r/20250801142822.180530-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf.c | 32 ++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index 35489fa81825..2ea81d81c0ae 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -47,9 +47,16 @@ static int pf_alloc_metadata(struct xe_gt *gt) static void pf_init_workers(struct xe_gt *gt) { + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); INIT_WORK(>->sriov.pf.workers.restart, pf_worker_restart_func); } +static void pf_fini_workers(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + disable_work_sync(>->sriov.pf.workers.restart); +} + /** * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF. * @gt: the &xe_gt to initialize @@ -79,6 +86,21 @@ int xe_gt_sriov_pf_init_early(struct xe_gt *gt) return 0; } +static void pf_fini_action(void *arg) +{ + struct xe_gt *gt = arg; + + pf_fini_workers(gt); +} + +static int pf_init_late(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + return devm_add_action_or_reset(xe->drm.dev, pf_fini_action, gt); +} + /** * xe_gt_sriov_pf_init - Prepare SR-IOV PF data structures on PF. * @gt: the &xe_gt to initialize @@ -95,7 +117,15 @@ int xe_gt_sriov_pf_init(struct xe_gt *gt) if (err) return err; - return xe_gt_sriov_pf_migration_init(gt); + err = xe_gt_sriov_pf_migration_init(gt); + if (err) + return err; + + err = pf_init_late(gt); + if (err) + return err; + + return 0; } static bool pf_needs_enable_ggtt_guest_update(struct xe_device *xe) From c6c86441c465ea440dfb5039f1c26e629a6fd64c Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 1 Aug 2025 16:28:21 +0200 Subject: [PATCH 051/169] drm/xe/pf: Make sure PF is ready to configure VFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PF driver might be resumed just to configure VFs, but since it is doing some asynchronous GuC reconfigurations after fresh reset, we should wait until all pending works are completed. This is especially important in case of LMEM provisioning, since we also need to update the LMTT and send invalidation requests to all GuCs, which are expected to be already in the VGT mode. Fixes: 68ae022278a1 ("drm/xe/pf: Force GuC virtualization mode") Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Cc: Jonathan Cavitt Link: https://lore.kernel.org/r/20250801142822.180530-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf.c | 25 +++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_sriov_pf.h | 1 + drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c | 4 ++- drivers/gpu/drm/xe/xe_pci_sriov.c | 7 +++++- drivers/gpu/drm/xe/xe_sriov_pf.c | 27 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_sriov_pf.h | 1 + 6 files changed, 63 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index 2ea81d81c0ae..bdbd15f3afe3 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -16,6 +16,7 @@ #include "xe_gt_sriov_pf_migration.h" #include "xe_gt_sriov_pf_service.h" #include "xe_gt_sriov_printk.h" +#include "xe_guc_submit.h" #include "xe_mmio.h" #include "xe_pm.h" @@ -260,3 +261,27 @@ void xe_gt_sriov_pf_restart(struct xe_gt *gt) { pf_queue_restart(gt); } + +static void pf_flush_restart(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + flush_work(>->sriov.pf.workers.restart); +} + +/** + * xe_gt_sriov_pf_wait_ready() - Wait until per-GT PF SR-IOV support is ready. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt) +{ + /* don't wait if there is another ongoing reset */ + if (xe_guc_read_stopped(>->uc.guc)) + return -EBUSY; + + pf_flush_restart(gt); + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h index e2b2ff8132dc..e7fde3f9937a 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h @@ -11,6 +11,7 @@ struct xe_gt; #ifdef CONFIG_PCI_IOV int xe_gt_sriov_pf_init_early(struct xe_gt *gt); int xe_gt_sriov_pf_init(struct xe_gt *gt); +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt); void xe_gt_sriov_pf_init_hw(struct xe_gt *gt); void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid); void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c index bf679b21f485..3ed245e04d0c 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c @@ -22,6 +22,7 @@ #include "xe_gt_sriov_pf_policy.h" #include "xe_gt_sriov_pf_service.h" #include "xe_pm.h" +#include "xe_sriov_pf.h" /* * /sys/kernel/debug/dri/0/ @@ -205,7 +206,8 @@ static int CONFIG##_set(void *data, u64 val) \ return -EOVERFLOW; \ \ xe_pm_runtime_get(xe); \ - err = xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ + err = xe_sriov_pf_wait_ready(xe) ?: \ + xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ xe_pm_runtime_put(xe); \ \ return err; \ diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 8813efdcafbb..447a7867eecb 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -12,6 +12,7 @@ #include "xe_pci_sriov.h" #include "xe_pm.h" #include "xe_sriov.h" +#include "xe_sriov_pf.h" #include "xe_sriov_pf_helpers.h" #include "xe_sriov_printk.h" @@ -138,6 +139,10 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) xe_assert(xe, num_vfs <= total_vfs); xe_sriov_dbg(xe, "enabling %u VF%s\n", num_vfs, str_plural(num_vfs)); + err = xe_sriov_pf_wait_ready(xe); + if (err) + goto out; + /* * We must hold additional reference to the runtime PM to keep PF in D0 * during VFs lifetime, as our VFs do not implement the PM capability. @@ -169,7 +174,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) failed: pf_unprovision_vfs(xe, num_vfs); xe_pm_runtime_put(xe); - +out: xe_sriov_notice(xe, "Failed to enable %u VF%s (%pe)\n", num_vfs, str_plural(num_vfs), ERR_PTR(err)); return err; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c index afbdd894bd6e..27ddf3cc80e9 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -9,6 +9,7 @@ #include "xe_assert.h" #include "xe_device.h" +#include "xe_gt_sriov_pf.h" #include "xe_module.h" #include "xe_sriov.h" #include "xe_sriov_pf.h" @@ -102,6 +103,32 @@ int xe_sriov_pf_init_early(struct xe_device *xe) return 0; } +/** + * xe_sriov_pf_wait_ready() - Wait until PF is ready to operate. + * @xe: the &xe_device to test + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_wait_ready(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int err; + + if (xe_device_wedged(xe)) + return -ECANCELED; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_wait_ready(gt); + if (err) + return err; + } + + return 0; +} + /** * xe_sriov_pf_print_vfs_summary - Print SR-IOV PF information. * @xe: the &xe_device to print info from diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.h b/drivers/gpu/drm/xe/xe_sriov_pf.h index c392c3fcf085..e3b34f8f5e04 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf.h @@ -15,6 +15,7 @@ struct xe_device; #ifdef CONFIG_PCI_IOV bool xe_sriov_pf_readiness(struct xe_device *xe); int xe_sriov_pf_init_early(struct xe_device *xe); +int xe_sriov_pf_wait_ready(struct xe_device *xe); void xe_sriov_pf_debugfs_register(struct xe_device *xe, struct dentry *root); void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p); #else From 9fd9f221440024b7451678898facfb34af054310 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 1 Aug 2025 16:28:22 +0200 Subject: [PATCH 052/169] drm/xe/pf: Don't resume device from restart worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PF's restart worker shouldn't attempt to resume the device on its own, since its goal is to finish PF and VFs reprovisioning on the recently reset GuC. Take extra RPM reference while scheduling a work and release it from the worker or when we cancel a work. Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250801142822.180530-4-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index bdbd15f3afe3..c4dda87b47cc 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -55,7 +55,12 @@ static void pf_init_workers(struct xe_gt *gt) static void pf_fini_workers(struct xe_gt *gt) { xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); - disable_work_sync(>->sriov.pf.workers.restart); + + if (disable_work_sync(>->sriov.pf.workers.restart)) { + xe_gt_sriov_dbg_verbose(gt, "pending restart disabled!\n"); + /* release an rpm reference taken on the worker's behalf */ + xe_pm_runtime_put(gt_to_xe(gt)); + } } /** @@ -207,8 +212,11 @@ static void pf_cancel_restart(struct xe_gt *gt) { xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); - if (cancel_work_sync(>->sriov.pf.workers.restart)) + if (cancel_work_sync(>->sriov.pf.workers.restart)) { xe_gt_sriov_dbg_verbose(gt, "pending restart canceled!\n"); + /* release an rpm reference taken on the worker's behalf */ + xe_pm_runtime_put(gt_to_xe(gt)); + } } /** @@ -226,9 +234,12 @@ static void pf_restart(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); - xe_pm_runtime_get(xe); + xe_gt_assert(gt, !xe_pm_runtime_suspended(xe)); + xe_gt_sriov_pf_config_restart(gt); xe_gt_sriov_pf_control_restart(gt); + + /* release an rpm reference taken on our behalf */ xe_pm_runtime_put(xe); xe_gt_sriov_dbg(gt, "restart completed\n"); @@ -247,8 +258,13 @@ static void pf_queue_restart(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(xe)); - if (!queue_work(xe->sriov.wq, >->sriov.pf.workers.restart)) + /* take an rpm reference on behalf of the worker */ + xe_pm_runtime_get_noresume(xe); + + if (!queue_work(xe->sriov.wq, >->sriov.pf.workers.restart)) { xe_gt_sriov_dbg(gt, "restart already in queue!\n"); + xe_pm_runtime_put(xe); + } } /** From 0521a868222ffe636bf202b6e9d29292c1e19c62 Mon Sep 17 00:00:00 2001 From: Simon Richter Date: Sat, 2 Aug 2025 11:40:36 +0900 Subject: [PATCH 053/169] Mark xe driver as BROKEN if kernel page size is not 4kB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This driver, for the time being, assumes that the kernel page size is 4kB, so it fails on loong64 and aarch64 with 16kB pages, and ppc64el with 64kB pages. Signed-off-by: Simon Richter Reviewed-by: Thomas Hellström Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Cc: stable@vger.kernel.org # v6.8+ Signed-off-by: Thomas Hellström Link: https://lore.kernel.org/r/20250802024152.3021-1-Simon.Richter@hogyros.de --- drivers/gpu/drm/xe/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig index 2bb2bc052120..714d5702dfd7 100644 --- a/drivers/gpu/drm/xe/Kconfig +++ b/drivers/gpu/drm/xe/Kconfig @@ -5,6 +5,7 @@ config DRM_XE depends on KUNIT || !KUNIT depends on INTEL_VSEC || !INTEL_VSEC depends on X86_PLATFORM_DEVICES || !(X86 && ACPI) + depends on PAGE_SIZE_4KB || COMPILE_TEST || BROKEN select INTERVAL_TREE # we need shmfs for the swappable backing store, and in particular # the shmem_readpage() which depends upon tmpfs From d47cc89d810d7ec19fbb34331551a3439da99c01 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:38 +0200 Subject: [PATCH 054/169] drm/xe/sa: Avoid caching GGTT address within the manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-virtualized resources require fixups after SRIOV VF migration. Caching GGTT references rather than re-computing them from the underlying Buffer Object is something we want to avoid, as such code would require additional fixup step and additional locking around all the places where the address is accessed. This change removes the cached GPU address from the Sub-Allocation Manager, and introduces a function which recomputes and returns the address instead. v2: renamed xe_sa_manager_gpu_addr(), added kerneldoc Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Matthew Brost Reviewed-by: Michal Wajdeczko Link: https://lore.kernel.org/r/20250802031045.1127138-2-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gt_debugfs.c | 5 +++-- drivers/gpu/drm/xe/xe_guc_buf.c | 2 +- drivers/gpu/drm/xe/xe_sa.c | 1 - drivers/gpu/drm/xe/xe_sa.h | 15 ++++++++++++++- drivers/gpu/drm/xe/xe_sa_types.h | 1 - drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 8d4b66ca38b7..bf3a67b5951c 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -29,6 +29,7 @@ #include "xe_pm.h" #include "xe_reg_sr.h" #include "xe_reg_whitelist.h" +#include "xe_sa.h" #include "xe_sriov.h" #include "xe_tuning.h" #include "xe_uc_debugfs.h" @@ -128,7 +129,7 @@ static int sa_info(struct xe_gt *gt, struct drm_printer *p) xe_pm_runtime_get(gt_to_xe(gt)); drm_suballoc_dump_debug_info(&tile->mem.kernel_bb_pool->base, p, - tile->mem.kernel_bb_pool->gpu_addr); + xe_sa_manager_gpu_addr(tile->mem.kernel_bb_pool)); xe_pm_runtime_put(gt_to_xe(gt)); return 0; @@ -152,7 +153,7 @@ static int sa_info_vf_ccs(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read"); drm_printf(p, "-------------------------\n"); - drm_suballoc_dump_debug_info(&bb_pool->base, p, bb_pool->gpu_addr); + drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool)); drm_puts(p, "\n"); } diff --git a/drivers/gpu/drm/xe/xe_guc_buf.c b/drivers/gpu/drm/xe/xe_guc_buf.c index 14a07dca48e7..502ca3a4ee60 100644 --- a/drivers/gpu/drm/xe/xe_guc_buf.c +++ b/drivers/gpu/drm/xe/xe_guc_buf.c @@ -164,7 +164,7 @@ u64 xe_guc_cache_gpu_addr_from_ptr(struct xe_guc_buf_cache *cache, const void *p if (offset < 0 || offset + size > cache->sam->base.size) return 0; - return cache->sam->gpu_addr + offset; + return xe_sa_manager_gpu_addr(cache->sam) + offset; } #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c index 1d43e183ca21..fedd017d6dd3 100644 --- a/drivers/gpu/drm/xe/xe_sa.c +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -69,7 +69,6 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3 } sa_manager->bo = bo; sa_manager->is_iomem = bo->vmap.is_iomem; - sa_manager->gpu_addr = xe_bo_ggtt_addr(bo); if (bo->vmap.is_iomem) { sa_manager->cpu_ptr = kvzalloc(managed_size, GFP_KERNEL); diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h index 1170ee5a81a8..99dbf0eea540 100644 --- a/drivers/gpu/drm/xe/xe_sa.h +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -7,6 +7,8 @@ #include #include + +#include "xe_bo.h" #include "xe_sa_types.h" struct dma_fence; @@ -43,9 +45,20 @@ to_xe_sa_manager(struct drm_suballoc_manager *mng) return container_of(mng, struct xe_sa_manager, base); } +/** + * xe_sa_manager_gpu_addr - Retrieve GPU address of a back storage BO + * within suballocator. + * @sa_manager: the &xe_sa_manager struct instance + * Return: GGTT address of the back storage BO. + */ +static inline u64 xe_sa_manager_gpu_addr(struct xe_sa_manager *sa_manager) +{ + return xe_bo_ggtt_addr(sa_manager->bo); +} + static inline u64 xe_sa_bo_gpu_addr(struct drm_suballoc *sa) { - return to_xe_sa_manager(sa->manager)->gpu_addr + + return xe_sa_manager_gpu_addr(to_xe_sa_manager(sa->manager)) + drm_suballoc_soffset(sa); } diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h index 2b070ff1292e..cb7238799dcb 100644 --- a/drivers/gpu/drm/xe/xe_sa_types.h +++ b/drivers/gpu/drm/xe/xe_sa_types.h @@ -12,7 +12,6 @@ struct xe_bo; struct xe_sa_manager { struct drm_suballoc_manager base; struct xe_bo *bo; - u64 gpu_addr; void *cpu_ptr; bool is_iomem; }; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index e363240a3455..f0ca2a9b2bb7 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -169,7 +169,7 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_tile_vf_ccs *ctx) static void ccs_rw_update_ring(struct xe_tile_vf_ccs *ctx) { struct xe_lrc *lrc = xe_migrate_lrc(ctx->migrate); - u64 addr = ctx->mem.ccs_bb_pool->gpu_addr; + u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); u32 dw[10], i = 0; dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; From f1193b864c08213d90e50c20ec3aaf8a413607b6 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:39 +0200 Subject: [PATCH 055/169] drm/xe/vf: Pause submissions during RESFIX fixups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While applying post-migration fixups to VF, GuC will not respond to any commands. This means submissions have no way of finishing. To avoid acquiring additional resources and then stalling on hardware access, pause the submission work. This will decrease the chance of depleting resources, and speed up the recovery. v2: Commented xe_irq_resume() call v3: Typo fix Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Reviewed-by: Michał Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-3-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gpu_scheduler.c | 13 ++++++++ drivers/gpu/drm/xe/xe_gpu_scheduler.h | 1 + drivers/gpu/drm/xe/xe_guc_submit.c | 35 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 2 ++ drivers/gpu/drm/xe/xe_sriov_vf.c | 46 +++++++++++++++++++++++++++ 5 files changed, 97 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c index 869b43a4151d..455ccaf17314 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.c +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -101,6 +101,19 @@ void xe_sched_submission_stop(struct xe_gpu_scheduler *sched) cancel_work_sync(&sched->work_process_msg); } +/** + * xe_sched_submission_stop_async - Stop further runs of submission tasks on a scheduler. + * @sched: the &xe_gpu_scheduler struct instance + * + * This call disables further runs of scheduling work queue. It does not wait + * for any in-progress runs to finish, only makes sure no further runs happen + * afterwards. + */ +void xe_sched_submission_stop_async(struct xe_gpu_scheduler *sched) +{ + drm_sched_wqueue_stop(&sched->base); +} + void xe_sched_submission_resume_tdr(struct xe_gpu_scheduler *sched) { drm_sched_resume_timeout(&sched->base, sched->base.timeout); diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.h b/drivers/gpu/drm/xe/xe_gpu_scheduler.h index 308061f0cf37..e548b2aed95a 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.h +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.h @@ -21,6 +21,7 @@ void xe_sched_fini(struct xe_gpu_scheduler *sched); void xe_sched_submission_start(struct xe_gpu_scheduler *sched); void xe_sched_submission_stop(struct xe_gpu_scheduler *sched); +void xe_sched_submission_stop_async(struct xe_gpu_scheduler *sched); void xe_sched_submission_resume_tdr(struct xe_gpu_scheduler *sched); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 679ae229cc82..ebc137be4de2 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1836,6 +1836,19 @@ void xe_guc_submit_stop(struct xe_guc *guc) } +/** + * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be disabled + */ +void xe_guc_submit_pause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_sched_submission_stop_async(&q->guc->sched); +} + static void guc_exec_queue_start(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; @@ -1876,6 +1889,28 @@ int xe_guc_submit_start(struct xe_guc *guc) return 0; } +static void guc_exec_queue_unpause(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + + xe_sched_submission_start(sched); +} + +/** + * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be enabled + */ +void xe_guc_submit_unpause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + guc_exec_queue_unpause(q); + + wake_up_all(&guc->ct.wq); +} + static struct xe_exec_queue * g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) { diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 8f64e799283b..ff44500f3da2 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -18,6 +18,8 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc); void xe_guc_submit_reset_wait(struct xe_guc *guc); void xe_guc_submit_stop(struct xe_guc *guc); int xe_guc_submit_start(struct xe_guc *guc); +void xe_guc_submit_pause(struct xe_guc *guc); +void xe_guc_submit_unpause(struct xe_guc *guc); void xe_guc_submit_wedge(struct xe_guc *guc); int xe_guc_read_stopped(struct xe_guc *guc); diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 26e243c28994..c66b17da1ce7 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -11,6 +11,8 @@ #include "xe_gt_sriov_printk.h" #include "xe_gt_sriov_vf.h" #include "xe_guc_ct.h" +#include "xe_guc_submit.h" +#include "xe_irq.h" #include "xe_pm.h" #include "xe_sriov.h" #include "xe_sriov_printk.h" @@ -147,6 +149,48 @@ void xe_sriov_vf_init_early(struct xe_device *xe) xe_sriov_info(xe, "migration not supported by this module version\n"); } +/** + * vf_post_migration_shutdown - Stop the driver activities after VF migration. + * @xe: the &xe_device struct instance + * + * After this VM is migrated and assigned to a new VF, it is running on a new + * hardware, and therefore many hardware-dependent states and related structures + * require fixups. Without fixups, the hardware cannot do any work, and therefore + * all GPU pipelines are stalled. + * Stop some of kernel activities to make the fixup process faster. + */ +static void vf_post_migration_shutdown(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + + for_each_gt(gt, xe, id) + xe_guc_submit_pause(>->uc.guc); +} + +/** + * vf_post_migration_kickstart - Re-start the driver activities under new hardware. + * @xe: the &xe_device struct instance + * + * After we have finished with all post-migration fixups, restart the driver + * activities to continue feeding the GPU with workloads. + */ +static void vf_post_migration_kickstart(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + + /* + * Make sure interrupts on the new HW are properly set. The GuC IRQ + * must be working at this point, since the recovery did started, + * but the rest was not enabled using the procedure from spec. + */ + xe_irq_resume(xe); + + for_each_gt(gt, xe, id) + xe_guc_submit_unpause(>->uc.guc); +} + static bool gt_vf_post_migration_needed(struct xe_gt *gt) { return test_bit(gt->info.id, >_to_xe(gt)->sriov.vf.migration.gt_flags); @@ -230,6 +274,7 @@ static void vf_post_migration_recovery(struct xe_device *xe) drm_dbg(&xe->drm, "migration recovery in progress\n"); xe_pm_runtime_get(xe); + vf_post_migration_shutdown(xe); if (!vf_migration_supported(xe)) { xe_sriov_err(xe, "migration not supported by this module version\n"); @@ -247,6 +292,7 @@ static void vf_post_migration_recovery(struct xe_device *xe) set_bit(id, &fixed_gts); } + vf_post_migration_kickstart(xe); err = vf_post_migration_notify_resfix_done(xe, fixed_gts); if (err) goto fail; From a0840b1ce95e37a1a870a3e794ed25c69080cecf Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:40 +0200 Subject: [PATCH 056/169] drm/xe: Block reset while recovering from VF migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resetting GuC during recovery could interfere with the recovery process. Such reset might be also triggered without justification, due to migration taking time, rather than due to the workload not progressing. Doing GuC reset during the recovery would cause exit of RESFIX state, and therefore continuation of GuC work while fixups are still being applied. To avoid that, reset needs to be blocked during the recovery. This patch blocks the reset during recovery. Reset request in that time range will be stalled, and unblocked only after GuC goes out of RESFIX state. In case a reset procedure already started while the recovery is triggered, there isn't much we can do - we cannot wait for it to finish as it involves waiting for hardware, and we can't be sure at which exact point of the reset procedure the GPU got switched. Therefore, the rare cases where migration happens while reset is in progress, are still dangerous. Resets are not a part of the standard flow, and cause unfinished workloads - that will happen during the reset interrupted by migration as well, so it doesn't diverge that much from what normally happens during such resets. v2: Introduce a new atomic for reset blocking, as we cannot reuse `stopped` atomic (that could lead to losing a workload). v3: Switched atomic functs to ones which include proper barriers Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Reviewed-by: Michal Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-4-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gt.c | 10 ++++++++ drivers/gpu/drm/xe/xe_guc_submit.c | 37 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 3 +++ drivers/gpu/drm/xe/xe_guc_types.h | 6 +++++ drivers/gpu/drm/xe/xe_sriov_vf.c | 12 ++++++++-- 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 5a79c6e3208b..390394bbaadc 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -41,6 +41,7 @@ #include "xe_gt_topology.h" #include "xe_guc_exec_queue_types.h" #include "xe_guc_pc.h" +#include "xe_guc_submit.h" #include "xe_hw_fence.h" #include "xe_hw_engine_class_sysfs.h" #include "xe_irq.h" @@ -802,6 +803,11 @@ static int do_gt_restart(struct xe_gt *gt) return 0; } +static int gt_wait_reset_unblock(struct xe_gt *gt) +{ + return xe_guc_wait_reset_unblock(>->uc.guc); +} + static int gt_reset(struct xe_gt *gt) { unsigned int fw_ref; @@ -816,6 +822,10 @@ static int gt_reset(struct xe_gt *gt) xe_gt_info(gt, "reset started\n"); + err = gt_wait_reset_unblock(gt); + if (!err) + xe_gt_warn(gt, "reset block failed to get lifted"); + xe_pm_runtime_get(gt_to_xe(gt)); if (xe_fault_inject_gt_reset()) { diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index ebc137be4de2..3274048e6a38 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1783,6 +1783,43 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) } } +/** + * xe_guc_submit_reset_block - Disallow reset calls on given GuC. + * @guc: the &xe_guc struct instance + */ +int xe_guc_submit_reset_block(struct xe_guc *guc) +{ + return atomic_fetch_or(1, &guc->submission_state.reset_blocked); +} + +/** + * xe_guc_submit_reset_unblock - Allow back reset calls on given GuC. + * @guc: the &xe_guc struct instance + */ +void xe_guc_submit_reset_unblock(struct xe_guc *guc) +{ + atomic_set_release(&guc->submission_state.reset_blocked, 0); + wake_up_all(&guc->ct.wq); +} + +static int guc_submit_reset_is_blocked(struct xe_guc *guc) +{ + return atomic_read_acquire(&guc->submission_state.reset_blocked); +} + +/* Maximum time of blocking reset */ +#define RESET_BLOCK_PERIOD_MAX (HZ * 5) + +/** + * xe_guc_wait_reset_unblock - Wait until reset blocking flag is lifted, or timeout. + * @guc: the &xe_guc struct instance + */ +int xe_guc_wait_reset_unblock(struct xe_guc *guc) +{ + return wait_event_timeout(guc->ct.wq, + !guc_submit_reset_is_blocked(guc), RESET_BLOCK_PERIOD_MAX); +} + int xe_guc_submit_reset_prepare(struct xe_guc *guc) { int ret; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index ff44500f3da2..4dd278f5f61b 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -20,6 +20,9 @@ void xe_guc_submit_stop(struct xe_guc *guc); int xe_guc_submit_start(struct xe_guc *guc); void xe_guc_submit_pause(struct xe_guc *guc); void xe_guc_submit_unpause(struct xe_guc *guc); +int xe_guc_submit_reset_block(struct xe_guc *guc); +void xe_guc_submit_reset_unblock(struct xe_guc *guc); +int xe_guc_wait_reset_unblock(struct xe_guc *guc); void xe_guc_submit_wedge(struct xe_guc *guc); int xe_guc_read_stopped(struct xe_guc *guc); diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h index 1fde7614fcc5..c7b9642b41ba 100644 --- a/drivers/gpu/drm/xe/xe_guc_types.h +++ b/drivers/gpu/drm/xe/xe_guc_types.h @@ -85,6 +85,12 @@ struct xe_guc { struct xarray exec_queue_lookup; /** @submission_state.stopped: submissions are stopped */ atomic_t stopped; + /** + * @submission_state.reset_blocked: reset attempts are blocked; + * blocking reset in order to delay it may be required if running + * an operation which is sensitive to resets. + */ + atomic_t reset_blocked; /** @submission_state.lock: protects submission state */ struct mutex lock; /** @submission_state.enabled: submission is enabled */ diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index c66b17da1ce7..71d28b30de43 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -163,9 +163,15 @@ static void vf_post_migration_shutdown(struct xe_device *xe) { struct xe_gt *gt; unsigned int id; + int ret = 0; - for_each_gt(gt, xe, id) + for_each_gt(gt, xe, id) { xe_guc_submit_pause(>->uc.guc); + ret |= xe_guc_submit_reset_block(>->uc.guc); + } + + if (ret) + drm_info(&xe->drm, "migration recovery encountered ongoing reset\n"); } /** @@ -187,8 +193,10 @@ static void vf_post_migration_kickstart(struct xe_device *xe) */ xe_irq_resume(xe); - for_each_gt(gt, xe, id) + for_each_gt(gt, xe, id) { + xe_guc_submit_reset_unblock(>->uc.guc); xe_guc_submit_unpause(>->uc.guc); + } } static bool gt_vf_post_migration_needed(struct xe_gt *gt) From b46ef766737c4ecc11aad9742fd7f70f8f922246 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:41 +0200 Subject: [PATCH 057/169] drm/xe/vf: Rebase HWSP of all contexts after migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All contexts require an update due to GGTT range shift, as that affects their HWSP. The HW status page of a context contains GGTT references, which need to be shifted to a new range (or re-computed using the previously updated vma nodes). The references include ring start address and indirect state address. v2: move some functions to better matched files v3: Add missing kerneldocs v4: Style fix Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Acked-by: Satyanarayana K V P Reviewed-by: Michal Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-5-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_exec_queue.c | 13 +++++++++++++ drivers/gpu/drm/xe/xe_exec_queue.h | 2 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 16 ++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 2 ++ drivers/gpu/drm/xe/xe_lrc.c | 17 +++++++++++++++++ drivers/gpu/drm/xe/xe_lrc.h | 1 + drivers/gpu/drm/xe/xe_sriov_vf.c | 1 + 7 files changed, 52 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index d5958518ef00..ae36a35ec80b 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1076,3 +1076,16 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) return err; } + +/** + * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references + * within all LRCs of a queue. + * @q: the &xe_exec_queue struct instance containing target LRCs + */ +void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q) +{ + int i; + + for (i = 0; i < q->width; ++i) + xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); +} diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 17bc50a7f05a..1d399a33c5c0 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -90,4 +90,6 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm); void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); +void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q); + #endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 3274048e6a38..f756b3855873 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2505,3 +2505,19 @@ void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) guc_exec_queue_print(q, p); mutex_unlock(&guc->submission_state.lock); } + +/** + * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all + * exec queues registered to given GuC. + * @guc: the &xe_guc struct instance + */ +void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_exec_queue_contexts_hwsp_rebase(q); + mutex_unlock(&guc->submission_state.lock); +} diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 4dd278f5f61b..d22f2d36dca1 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -46,4 +46,6 @@ xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapsh void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); void xe_guc_register_exec_queue(struct xe_exec_queue *q, int ctx_type); +void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc); + #endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 540f044bf425..baf926ad3a66 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1439,6 +1439,23 @@ void xe_lrc_destroy(struct kref *ref) kfree(lrc); } +/** + * xe_lrc_update_hwctx_regs_with_address - Re-compute GGTT references within given LRC. + * @lrc: the &xe_lrc struct instance + */ +void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) { + xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, + __xe_lrc_indirect_ring_ggtt_addr(lrc)); + + xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, + __xe_lrc_ring_ggtt_addr(lrc)); + } else { + xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); + } +} + void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) { if (xe_lrc_has_indirect_ring_state(lrc)) diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index b6c8053c581b..cc231bec709c 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -88,6 +88,7 @@ bool xe_lrc_ring_is_idle(struct xe_lrc *lrc); u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc); u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc); u32 *xe_lrc_regs(struct xe_lrc *lrc); +void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc); u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr); void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val); diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 71d28b30de43..54a6218e48b0 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -269,6 +269,7 @@ static int gt_vf_post_migration_fixups(struct xe_gt *gt) shift = xe_gt_sriov_vf_ggtt_shift(gt); if (shift) { xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift); + xe_guc_contexts_hwsp_rebase(>->uc.guc); /* FIXME: add the recovery steps */ xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift); } From 30d137ddcee514d3ee5a498713628bd50d41e87b Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:42 +0200 Subject: [PATCH 058/169] drm/xe/vf: Rebase MEMIRQ structures for all contexts after migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All contexts require an update of state data, as the data includes GGTT references to memirq-related buffers. Default contexts need these references updated as well, because they are not refreshed when a new context is created from them. The way we write to vram requires scratch buffer to be used before the whole block is memcopied. Since using kalloc() within specific recovery functions would lead to unintended relations between locks, we are allocating the buffer earlier, before any locks are taken. The same buffer will be used for other steps of the recovery. v2: Update addresses by xe_lrc_write_ctx_reg() rather than set_memory_based_intr() v3: Renamed parameter, reordered parameters in some functs v4: Check if have MEMIRQ, move `xe_gt*` funct to proper file v5: Revert back to requiring scratch buffer, but allocate it earlier this time Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Acked-by: Satyanarayana K V P Reviewed-by: Michal Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-6-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 ++-- drivers/gpu/drm/xe/xe_exec_queue.h | 2 +- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 14 ++++++++ drivers/gpu/drm/xe/xe_gt_sriov_vf.h | 1 + drivers/gpu/drm/xe/xe_guc_submit.c | 5 +-- drivers/gpu/drm/xe/xe_guc_submit.h | 2 +- drivers/gpu/drm/xe/xe_lrc.c | 51 +++++++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_lrc.h | 4 +++ drivers/gpu/drm/xe/xe_sriov_vf.c | 20 +++++++++-- 9 files changed, 96 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index ae36a35ec80b..0beb6388acb0 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1081,11 +1081,14 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references * within all LRCs of a queue. * @q: the &xe_exec_queue struct instance containing target LRCs + * @scratch: scratch buffer to be used as temporary storage */ -void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q) +void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) { int i; - for (i = 0; i < q->width; ++i) + for (i = 0; i < q->width; ++i) { + xe_lrc_update_memirq_regs_with_address(q->lrc[i], q->hwe, scratch); xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); + } } diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 1d399a33c5c0..da720197929b 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -90,6 +90,6 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm); void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); -void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q); +void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); #endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index b282838d59e6..0461d5513487 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -25,6 +25,7 @@ #include "xe_guc.h" #include "xe_guc_hxg_helpers.h" #include "xe_guc_relay.h" +#include "xe_lrc.h" #include "xe_mmio.h" #include "xe_sriov.h" #include "xe_sriov_vf.h" @@ -750,6 +751,19 @@ failed: return err; } +/** + * xe_gt_sriov_vf_default_lrcs_hwsp_rebase - Update GGTT references in HWSP of default LRCs. + * @gt: the &xe_gt struct instance + */ +void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) + xe_default_lrc_update_memirq_regs_with_address(hwe); +} + /** * xe_gt_sriov_vf_migrated_event_handler - Start a VF migration recovery, * or just mark that a GuC is ready for it. diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h index e0357f341a2d..0af1dc769fe0 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h @@ -21,6 +21,7 @@ void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, int xe_gt_sriov_vf_query_config(struct xe_gt *gt); int xe_gt_sriov_vf_connect(struct xe_gt *gt); int xe_gt_sriov_vf_query_runtime(struct xe_gt *gt); +void xe_gt_sriov_vf_default_lrcs_hwsp_rebase(struct xe_gt *gt); int xe_gt_sriov_vf_notify_resfix_done(struct xe_gt *gt); void xe_gt_sriov_vf_migrated_event_handler(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index f756b3855873..300dac6dab31 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2510,14 +2510,15 @@ void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all * exec queues registered to given GuC. * @guc: the &xe_guc struct instance + * @scratch: scratch buffer to be used as temporary storage */ -void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc) +void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) { struct xe_exec_queue *q; unsigned long index; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - xe_exec_queue_contexts_hwsp_rebase(q); + xe_exec_queue_contexts_hwsp_rebase(q, scratch); mutex_unlock(&guc->submission_state.lock); } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index d22f2d36dca1..9a2718c81d43 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -46,6 +46,6 @@ xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapsh void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); void xe_guc_register_exec_queue(struct xe_exec_queue *q, int ctx_type); -void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc); +void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); #endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index baf926ad3a66..9a67f4ae235a 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -697,7 +697,13 @@ u32 xe_lrc_regs_offset(struct xe_lrc *lrc) return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; } -static size_t lrc_reg_size(struct xe_device *xe) +/** + * xe_lrc_reg_size() - Get size of the LRC registers area within queues + * @xe: the &xe_device struct instance + * + * Returns: Size of the LRC registers area for current platform + */ +size_t xe_lrc_reg_size(struct xe_device *xe) { if (GRAPHICS_VERx100(xe) >= 1250) return 96 * sizeof(u32); @@ -707,7 +713,7 @@ static size_t lrc_reg_size(struct xe_device *xe) size_t xe_lrc_skip_size(struct xe_device *xe) { - return LRC_PPHWSP_SIZE + lrc_reg_size(xe); + return LRC_PPHWSP_SIZE + xe_lrc_reg_size(xe); } static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) @@ -948,6 +954,47 @@ static void *empty_lrc_data(struct xe_hw_engine *hwe) return data; } +/** + * xe_default_lrc_update_memirq_regs_with_address - Re-compute GGTT references in default LRC + * of given engine. + * @hwe: the &xe_hw_engine struct instance + */ +void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + u32 *regs; + + if (!gt->default_lrc[hwe->class]) + return; + + regs = gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE; + set_memory_based_intr(regs, hwe); +} + +/** + * xe_lrc_update_memirq_regs_with_address - Re-compute GGTT references in mem interrupt data + * for given LRC. + * @lrc: the &xe_lrc struct instance + * @hwe: the &xe_hw_engine struct instance + * @regs: scratch buffer to be used as temporary storage + */ +void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *regs) +{ + struct xe_gt *gt = hwe->gt; + struct iosys_map map; + size_t regs_len; + + if (!xe_device_uses_memirq(gt_to_xe(gt))) + return; + + map = __xe_lrc_regs_map(lrc); + regs_len = xe_lrc_reg_size(gt_to_xe(gt)); + xe_map_memcpy_from(gt_to_xe(gt), regs, &map, 0, regs_len); + set_memory_based_intr(regs, hwe); + xe_map_memcpy_to(gt_to_xe(gt), &map, 0, regs, regs_len); +} + static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) { u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index cc231bec709c..eceeeee6c021 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -89,6 +89,9 @@ u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc); u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc); u32 *xe_lrc_regs(struct xe_lrc *lrc); void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc); +void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe); +void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *regs); u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr); void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val); @@ -107,6 +110,7 @@ s32 xe_lrc_start_seqno(struct xe_lrc *lrc); u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc); struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc); +size_t xe_lrc_reg_size(struct xe_device *xe); size_t xe_lrc_skip_size(struct xe_device *xe); void xe_lrc_dump_default(struct drm_printer *p, diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 54a6218e48b0..43ac73e432d4 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -13,6 +13,7 @@ #include "xe_guc_ct.h" #include "xe_guc_submit.h" #include "xe_irq.h" +#include "xe_lrc.h" #include "xe_pm.h" #include "xe_sriov.h" #include "xe_sriov_printk.h" @@ -244,6 +245,11 @@ static int vf_get_next_migrated_gt_id(struct xe_device *xe) return -1; } +static size_t post_migration_scratch_size(struct xe_device *xe) +{ + return xe_lrc_reg_size(xe); +} + /** * Perform post-migration fixups on a single GT. * @@ -260,19 +266,29 @@ static int vf_get_next_migrated_gt_id(struct xe_device *xe) static int gt_vf_post_migration_fixups(struct xe_gt *gt) { s64 shift; + void *buf; int err; + buf = kmalloc(post_migration_scratch_size(gt_to_xe(gt)), GFP_KERNEL); + if (!buf) + return -ENOMEM; + err = xe_gt_sriov_vf_query_config(gt); - if (err) + if (err) { + kfree(buf); return err; + } shift = xe_gt_sriov_vf_ggtt_shift(gt); if (shift) { xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift); - xe_guc_contexts_hwsp_rebase(>->uc.guc); + xe_gt_sriov_vf_default_lrcs_hwsp_rebase(gt); + xe_guc_contexts_hwsp_rebase(>->uc.guc, buf); /* FIXME: add the recovery steps */ xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift); } + + kfree(buf); return 0; } From a0dda25d24e636df5c30a9370464b7cebc709faf Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:43 +0200 Subject: [PATCH 059/169] drm/xe/vf: Post migration, repopulate ring area for pending request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commands within ring area allocated for a request may contain references to GGTT. These references require update after VF migration, in order to continue any preempted LRCs, or jobs which were emitted to the ring but not sent to GuC yet. This change calls the emit function again for all such jobs, as part of post-migration recovery. v2: Moved few functions to better files v3: Take job_list_lock v4: Rephrased comments Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Reviewed-by: Michal Winiarski Cc: Jonathan Cavitt Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250802031045.1127138-7-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_exec_queue.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_exec_queue.h | 2 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 2 ++ drivers/gpu/drm/xe/xe_sriov_vf.c | 2 +- 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 0beb6388acb0..05a1bd6b55bf 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1092,3 +1092,27 @@ void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); } } + +/** + * xe_exec_queue_jobs_ring_restore - Re-emit ring commands of requests pending on given queue. + * @q: the &xe_exec_queue struct instance + */ +void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q) +{ + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_sched_job *job; + + /* + * This routine is used within VF migration recovery. This means + * using the lock here introduces a restriction: we cannot wait + * for any GFX HW response while the lock is taken. + */ + spin_lock(&sched->base.job_list_lock); + list_for_each_entry(job, &sched->base.pending_list, drm.list) { + if (xe_sched_job_is_error(job)) + continue; + + q->ring_ops->emit_job(job); + } + spin_unlock(&sched->base.job_list_lock); +} diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index da720197929b..0ffc0cb03aa6 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -92,4 +92,6 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); +void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q); + #endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 300dac6dab31..7a54890ce9d0 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -781,6 +781,30 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) return fence; } +/** + * xe_guc_jobs_ring_rebase - Re-emit ring commands of requests pending + * on all queues under a guc. + * @guc: the &xe_guc struct instance + */ +void xe_guc_jobs_ring_rebase(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + /* + * This routine is used within VF migration recovery. This means + * using the lock here introduces a restriction: we cannot wait + * for any GFX HW response while the lock is taken. + */ + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + if (exec_queue_killed_or_banned_or_wedged(q)) + continue; + xe_exec_queue_jobs_ring_restore(q); + } + mutex_unlock(&guc->submission_state.lock); +} + static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 9a2718c81d43..92a6f0ade615 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -34,6 +34,8 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len); +void xe_guc_jobs_ring_rebase(struct xe_guc *guc); + struct xe_guc_submit_exec_queue_snapshot * xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q); void diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 43ac73e432d4..a219395c15de 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -284,7 +284,7 @@ static int gt_vf_post_migration_fixups(struct xe_gt *gt) xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift); xe_gt_sriov_vf_default_lrcs_hwsp_rebase(gt); xe_guc_contexts_hwsp_rebase(>->uc.guc, buf); - /* FIXME: add the recovery steps */ + xe_guc_jobs_ring_rebase(>->uc.guc); xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift); } From 168b5867318bff761c9095d12115c302c483d66a Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:44 +0200 Subject: [PATCH 060/169] drm/xe/vf: Refresh utilization buffer during migration recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WA buffer we use to capture context utilization contains GGTT references. This means its instructions have to be either fixed or re-emitted during VF post-migration recovery. This patch adds re-emitting content of the utilization WA BB during the recovery. The way we write to vram requires scratch buffer to be used before the whole block is memcopied. We are re-using a scratch buffer introduced in earlier part of the recovery. This is not a performance optimization, but a necessity to avoid creating dependencies between locks. v2: Notable rebase after "Prepare WA BB setup for more users" patch v3: Added error propagation Signed-off-by: Tomasz Lis Cc: Michal Wajdeczko Cc: Michal Winiarski Reviewed-by: Michal Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-8-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_exec_queue.c | 10 +++++++- drivers/gpu/drm/xe/xe_exec_queue.h | 2 +- drivers/gpu/drm/xe/xe_guc_submit.c | 14 ++++++++--- drivers/gpu/drm/xe/xe_guc_submit.h | 2 +- drivers/gpu/drm/xe/xe_lrc.c | 39 ++++++++++++++++++++++++------ drivers/gpu/drm/xe/xe_lrc.h | 4 +++ drivers/gpu/drm/xe/xe_sriov_vf.c | 15 ++++++------ 7 files changed, 66 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 05a1bd6b55bf..6c176183ed58 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1082,15 +1082,23 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) * within all LRCs of a queue. * @q: the &xe_exec_queue struct instance containing target LRCs * @scratch: scratch buffer to be used as temporary storage + * + * Returns: zero on success, negative error code on failure */ -void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) +int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) { int i; + int err = 0; for (i = 0; i < q->width; ++i) { xe_lrc_update_memirq_regs_with_address(q->lrc[i], q->hwe, scratch); xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); + err = xe_lrc_setup_wa_bb_with_scratch(q->lrc[i], q->hwe, scratch); + if (err) + break; } + + return err; } /** diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 0ffc0cb03aa6..4d416f23001c 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -90,7 +90,7 @@ int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm); void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); -void xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); +int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 7a54890ce9d0..20f5b9820264 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2535,14 +2535,22 @@ void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) * exec queues registered to given GuC. * @guc: the &xe_guc struct instance * @scratch: scratch buffer to be used as temporary storage + * + * Returns: zero on success, negative error code on failure. */ -void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) +int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) { struct xe_exec_queue *q; unsigned long index; + int err = 0; mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - xe_exec_queue_contexts_hwsp_rebase(q, scratch); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); + if (err) + break; + } mutex_unlock(&guc->submission_state.lock); + + return err; } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 92a6f0ade615..6b5df5d0956b 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -48,6 +48,6 @@ xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapsh void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); void xe_guc_register_exec_queue(struct xe_exec_queue *q, int ctx_type); -void xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); +int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); #endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 9a67f4ae235a..8760c4c2ca38 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -41,7 +41,6 @@ #define LRC_PPHWSP_SIZE SZ_4K #define LRC_INDIRECT_CTX_BO_SIZE SZ_4K #define LRC_INDIRECT_RING_STATE_SIZE SZ_4K -#define LRC_WA_BB_SIZE SZ_4K /* * Layout of the LRC and associated data allocated as @@ -1149,13 +1148,11 @@ static int setup_bo(struct bo_setup_state *state) ssize_t remain; if (state->lrc->bo->vmap.is_iomem) { - state->buffer = kmalloc(state->max_size, GFP_KERNEL); if (!state->buffer) return -ENOMEM; state->ptr = state->buffer; } else { state->ptr = state->lrc->bo->vmap.vaddr + state->offset; - state->buffer = NULL; } remain = state->max_size / sizeof(u32); @@ -1180,7 +1177,6 @@ static int setup_bo(struct bo_setup_state *state) return 0; fail: - kfree(state->buffer); return -ENOSPC; } @@ -1192,10 +1188,16 @@ static void finish_bo(struct bo_setup_state *state) xe_map_memcpy_to(gt_to_xe(state->lrc->gt), &state->lrc->bo->vmap, state->offset, state->buffer, state->written * sizeof(u32)); - kfree(state->buffer); } -static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) +/** + * xe_lrc_setup_wa_bb_with_scratch - Execute all wa bb setup callbacks. + * @lrc: the &xe_lrc struct instance + * @hwe: the &xe_hw_engine struct instance + * @scratch: preallocated scratch buffer for temporary storage + * Return: 0 on success, negative error code on failure + */ +int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *scratch) { static const struct bo_setup funcs[] = { { .setup = setup_timestamp_wa }, @@ -1206,6 +1208,7 @@ static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) .lrc = lrc, .hwe = hwe, .max_size = LRC_WA_BB_SIZE, + .buffer = scratch, .reserve_dw = 1, .offset = __xe_lrc_wa_bb_offset(lrc), .funcs = funcs, @@ -1228,6 +1231,21 @@ static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) return 0; } +static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) +{ + u32 *buf = NULL; + int ret; + + if (lrc->bo->vmap.is_iomem) + buf = kmalloc(LRC_WA_BB_SIZE, GFP_KERNEL); + + ret = xe_lrc_setup_wa_bb_with_scratch(lrc, hwe, buf); + + kfree(buf); + + return ret; +} + static int setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) { @@ -1238,6 +1256,7 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) .lrc = lrc, .hwe = hwe, .max_size = (63 * 64) /* max 63 cachelines */, + .buffer = NULL, .offset = __xe_lrc_indirect_ctx_offset(lrc), }; int ret; @@ -1254,9 +1273,14 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) if (xe_gt_WARN_ON(lrc->gt, !state.funcs)) return 0; + if (lrc->bo->vmap.is_iomem) + state.buffer = kmalloc(state.max_size, GFP_KERNEL); + ret = setup_bo(&state); - if (ret) + if (ret) { + kfree(state.buffer); return ret; + } /* * Align to 64B cacheline so there's no garbage at the end for CS to @@ -1268,6 +1292,7 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) } finish_bo(&state); + kfree(state.buffer); xe_lrc_write_ctx_reg(lrc, CTX_CS_INDIRECT_CTX, diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index eceeeee6c021..188565465779 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -42,6 +42,8 @@ struct xe_lrc_snapshot { #define LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR (0x34 * 4) #define LRC_PPHWSP_PXP_INVAL_SCRATCH_ADDR (0x40 * 4) +#define LRC_WA_BB_SIZE SZ_4K + #define XE_LRC_CREATE_RUNALONE 0x1 #define XE_LRC_CREATE_PXP 0x2 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, @@ -129,6 +131,8 @@ u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc); u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc); u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc); u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc); +int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + u32 *scratch); /** * xe_lrc_update_timestamp - readout LRC timestamp and update cached value diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index a219395c15de..5de81f213d83 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -247,7 +247,7 @@ static int vf_get_next_migrated_gt_id(struct xe_device *xe) static size_t post_migration_scratch_size(struct xe_device *xe) { - return xe_lrc_reg_size(xe); + return max(xe_lrc_reg_size(xe), LRC_WA_BB_SIZE); } /** @@ -274,22 +274,23 @@ static int gt_vf_post_migration_fixups(struct xe_gt *gt) return -ENOMEM; err = xe_gt_sriov_vf_query_config(gt); - if (err) { - kfree(buf); - return err; - } + if (err) + goto out; shift = xe_gt_sriov_vf_ggtt_shift(gt); if (shift) { xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift); xe_gt_sriov_vf_default_lrcs_hwsp_rebase(gt); - xe_guc_contexts_hwsp_rebase(>->uc.guc, buf); + err = xe_guc_contexts_hwsp_rebase(>->uc.guc, buf); + if (err) + goto out; xe_guc_jobs_ring_rebase(>->uc.guc); xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift); } +out: kfree(buf); - return 0; + return err; } static void vf_post_migration_recovery(struct xe_device *xe) From ba180a362128cb71d16c3f0ce6645448011d2607 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Sat, 2 Aug 2025 05:10:45 +0200 Subject: [PATCH 061/169] drm/xe/vf: Rebase exec queue parallel commands during migration recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel exec queues have an additional command streamer buffer which holds a GGTT reference to data within context status. The GGTT references have to be fixed after VF migration. v2: Properly handle nop entry, verify if parsing goes ok v3: Improve error/warn logging, add propagation of errors, give names to magic offsets Signed-off-by: Tomasz Lis Cc: Michal Winiarski Reviewed-by: Michał Winiarski Link: https://lore.kernel.org/r/20250802031045.1127138-9-tomasz.lis@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/abi/guc_actions_abi.h | 8 ++++ drivers/gpu/drm/xe/xe_guc_submit.c | 54 ++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h index 81eb046aeebf..d8cf68a0516d 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -193,6 +193,14 @@ enum xe_guc_register_context_multi_lrc_param_offsets { XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN = 11, }; +enum xe_guc_context_wq_item_offsets { + XE_GUC_CONTEXT_WQ_HEADER_DATA_0_TYPE_LEN = 0, + XE_GUC_CONTEXT_WQ_EL_INFO_DATA_1_CTX_DESC_LOW, + XE_GUC_CONTEXT_WQ_EL_INFO_DATA_2_GUCCTX_RINGTAIL_FREEZEPOCS, + XE_GUC_CONTEXT_WQ_EL_INFO_DATA_3_WI_FENCE_ID, + XE_GUC_CONTEXT_WQ_EL_CHILD_LIST_DATA_4_RINGTAIL, +}; + enum xe_guc_report_status { XE_GUC_REPORT_STATUS_UNKNOWN = 0x0, XE_GUC_REPORT_STATUS_ACKED = 0x1, diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 20f5b9820264..554a25ca77d3 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -671,12 +671,18 @@ static void wq_item_append(struct xe_exec_queue *q) if (wq_wait_for_space(q, wqi_size)) return; + xe_gt_assert(guc_to_gt(guc), i == XE_GUC_CONTEXT_WQ_HEADER_DATA_0_TYPE_LEN); wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | FIELD_PREP(WQ_LEN_MASK, len_dw); + xe_gt_assert(guc_to_gt(guc), i == XE_GUC_CONTEXT_WQ_EL_INFO_DATA_1_CTX_DESC_LOW); wqi[i++] = xe_lrc_descriptor(q->lrc[0]); + xe_gt_assert(guc_to_gt(guc), i == + XE_GUC_CONTEXT_WQ_EL_INFO_DATA_2_GUCCTX_RINGTAIL_FREEZEPOCS); wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); + xe_gt_assert(guc_to_gt(guc), i == XE_GUC_CONTEXT_WQ_EL_INFO_DATA_3_WI_FENCE_ID); wqi[i++] = 0; + xe_gt_assert(guc_to_gt(guc), i == XE_GUC_CONTEXT_WQ_EL_CHILD_LIST_DATA_4_RINGTAIL); for (j = 1; j < q->width; ++j) { struct xe_lrc *lrc = q->lrc[j]; @@ -697,6 +703,50 @@ static void wq_item_append(struct xe_exec_queue *q) parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); } +static int wq_items_rebase(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + int i = q->guc->wqi_head; + + /* the ring starts after a header struct */ + iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, wq[0])); + + while ((i % WQ_SIZE) != (q->guc->wqi_tail % WQ_SIZE)) { + u32 len_dw, type, val; + + if (drm_WARN_ON_ONCE(&xe->drm, i < 0 || i > 2 * WQ_SIZE)) + break; + + val = xe_map_rd_ring_u32(xe, &map, i / sizeof(u32) + + XE_GUC_CONTEXT_WQ_HEADER_DATA_0_TYPE_LEN, + WQ_SIZE / sizeof(u32)); + len_dw = FIELD_GET(WQ_LEN_MASK, val); + type = FIELD_GET(WQ_TYPE_MASK, val); + + if (drm_WARN_ON_ONCE(&xe->drm, len_dw >= WQ_SIZE / sizeof(u32))) + break; + + if (type == WQ_TYPE_MULTI_LRC) { + val = xe_lrc_descriptor(q->lrc[0]); + xe_map_wr_ring_u32(xe, &map, i / sizeof(u32) + + XE_GUC_CONTEXT_WQ_EL_INFO_DATA_1_CTX_DESC_LOW, + WQ_SIZE / sizeof(u32), val); + } else if (drm_WARN_ON_ONCE(&xe->drm, type != WQ_TYPE_NOOP)) { + break; + } + + i += (len_dw + 1) * sizeof(u32); + } + + if ((i % WQ_SIZE) != (q->guc->wqi_tail % WQ_SIZE)) { + xe_gt_err(q->gt, "Exec queue fixups incomplete - wqi parse failed\n"); + return -EBADMSG; + } + return 0; +} + #define RESUME_PENDING ~0x0ull static void submit_exec_queue(struct xe_exec_queue *q) { @@ -2549,6 +2599,10 @@ int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); if (err) break; + if (xe_exec_queue_is_parallel(q)) + err = wq_items_rebase(q); + if (err) + break; } mutex_unlock(&guc->submission_state.lock); From bcddb12c027434fdf0491c1a05a3fe4fd2263d71 Mon Sep 17 00:00:00 2001 From: Tangudu Tilak Tirumalesh Date: Thu, 31 Jul 2025 22:01:44 +0000 Subject: [PATCH 062/169] drm/xe: Extend wa_13012615864 to additional Xe2 and Xe3 platforms Extend WA 13012615864 to Graphics Versions 20.01,20.02,20.04 and 30.03. Signed-off-by: Tangudu Tilak Tirumalesh Signed-off-by: Jonathan Cavitt Cc: Matt Roper Cc: Michal Wajdeczko Cc: Rodrigo Vivi Reviewed-by: Gustavo Sousa Link: https://lore.kernel.org/r/20250731220143.72942-2-jonathan.cavitt@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_wa.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 22a98600fd8f..535067e7fb0c 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -538,6 +538,11 @@ static const struct xe_rtp_entry_sr engine_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE)) }, + { XE_RTP_NAME("13012615864"), + XE_RTP_RULES(GRAPHICS_VERSION(2004), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS)) + }, /* Xe2_HPG */ @@ -602,6 +607,11 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, STK_ID_RESTRICT)) }, + { XE_RTP_NAME("13012615864"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS)) + }, /* Xe2_LPM */ @@ -647,7 +657,8 @@ static const struct xe_rtp_entry_sr engine_was[] = { XE_RTP_ACTIONS(SET(TDL_CHICKEN, QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE)) }, { XE_RTP_NAME("13012615864"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001), OR, + GRAPHICS_VERSION(3003), FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, RES_CHK_SPR_DIS)) }, From 7c9de25efa7677acd4feae905165571be05b6f29 Mon Sep 17 00:00:00 2001 From: Jonathan Cavitt Date: Mon, 28 Jul 2025 19:48:06 +0000 Subject: [PATCH 063/169] drm/xe/xe_guc_ads: Consolidate guc_waklv_enable functions Presently, multiple versions of the guc_waklv_enable_.* function exist, all with different numbers of dwords added to the klv_entry array. This is not extensible, and more duplicates of the function will need to be created if it ever becomes necessary to support 3 or more dwords per wa in the future. Consolidate the disparate guc_waklv_enable functions into a single guc_waklv_enable function that can take an arbitrary number of dword values. v2: - Update length value properly (Shuicheng) v3: (Harrison) - Use data as a term instead of dwords or arr - Reformat warning message to use hex values - Eliminate need for kzalloc and klv_entry array - Reorder function parameters to fix line wrapping v4: - Miscellaneous formatting fixes (Cavitt) v5: (Harrison) - s/data_range/data_len_dw - Use data_len_dw to calculate size for xe_map_memcpy_to Suggested-by: Daniele Ceraolo Spurio Signed-off-by: Jonathan Cavitt Cc: Lucas De Marchi Cc: Shuicheng Lin Cc: John Harrison Reviewed-by: John Harrison Link: https://lore.kernel.org/r/20250728194806.68176-2-jonathan.cavitt@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_guc_ads.c | 138 +++++++++----------------------- 1 file changed, 40 insertions(+), 98 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 8ff8626227ae..2ceaa197cb2f 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -284,81 +284,26 @@ static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) return total_size; } -static void guc_waklv_enable_two_word(struct xe_guc_ads *ads, - enum xe_guc_klv_ids klv_id, - u32 value1, - u32 value2, - u32 *offset, u32 *remain) +static void guc_waklv_enable(struct xe_guc_ads *ads, + u32 data[], u32 data_len_dw, + u32 *offset, u32 *remain, + enum xe_guc_klv_ids klv_id) { - u32 size; - u32 klv_entry[] = { - /* 16:16 key/length */ - FIELD_PREP(GUC_KLV_0_KEY, klv_id) | - FIELD_PREP(GUC_KLV_0_LEN, 2), - value1, - value2, - /* 2 dword data */ - }; - - size = sizeof(klv_entry); + size_t size = sizeof(u32) * (1 + data_len_dw); if (*remain < size) { drm_warn(&ads_to_xe(ads)->drm, - "w/a klv buffer too small to add klv id %d\n", klv_id); - } else { - xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, - klv_entry, size); - *offset += size; - *remain -= size; - } -} - -static void guc_waklv_enable_one_word(struct xe_guc_ads *ads, - enum xe_guc_klv_ids klv_id, - u32 value, - u32 *offset, u32 *remain) -{ - u32 size; - u32 klv_entry[] = { - /* 16:16 key/length */ - FIELD_PREP(GUC_KLV_0_KEY, klv_id) | - FIELD_PREP(GUC_KLV_0_LEN, 1), - value, - /* 1 dword data */ - }; - - size = sizeof(klv_entry); - - if (*remain < size) { - drm_warn(&ads_to_xe(ads)->drm, - "w/a klv buffer too small to add klv id %d\n", klv_id); - } else { - xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, - klv_entry, size); - *offset += size; - *remain -= size; - } -} - -static void guc_waklv_enable_simple(struct xe_guc_ads *ads, - enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain) -{ - u32 klv_entry[] = { - /* 16:16 key/length */ - FIELD_PREP(GUC_KLV_0_KEY, klv_id) | - FIELD_PREP(GUC_KLV_0_LEN, 0), - /* 0 dwords data */ - }; - u32 size; - - size = sizeof(klv_entry); - - if (xe_gt_WARN(ads_to_gt(ads), *remain < size, - "w/a klv buffer too small to add klv id %d\n", klv_id)) + "w/a klv buffer too small to add klv id 0x%04X\n", klv_id); return; + } + + /* 16:16 key/length */ + xe_map_wr(ads_to_xe(ads), ads_to_map(ads), *offset, u32, + FIELD_PREP(GUC_KLV_0_KEY, klv_id) | FIELD_PREP(GUC_KLV_0_LEN, data_len_dw)); + /* data_len_dw dwords of data */ + xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), + *offset + sizeof(u32), data, data_len_dw * sizeof(u32)); - xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, - klv_entry, size); *offset += size; *remain -= size; } @@ -373,49 +318,46 @@ static void guc_waklv_init(struct xe_guc_ads *ads) remain = guc_ads_waklv_size(ads); if (XE_WA(gt, 14019882105) || XE_WA(gt, 16021333562)) - guc_waklv_enable_simple(ads, - GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); if (XE_WA(gt, 18024947630)) - guc_waklv_enable_simple(ads, - GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING); if (XE_WA(gt, 16022287689)) - guc_waklv_enable_simple(ads, - GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE); if (XE_WA(gt, 14022866841)) - guc_waklv_enable_simple(ads, - GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO); /* * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now, * the default value for this register is determined to be 0xC40. This could change in the * future, so GuC depends on KMD to send it the correct value. */ - if (XE_WA(gt, 13011645652)) - guc_waklv_enable_one_word(ads, - GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE, - 0xC40, - &offset, &remain); + if (XE_WA(gt, 13011645652)) { + u32 data = 0xC40; + + guc_waklv_enable(ads, &data, sizeof(data) / sizeof(u32), &offset, &remain, + GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE); + } if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406)) - guc_waklv_enable_simple(ads, - GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_WA(gt, 16026508708)) - guc_waklv_enable_simple(ads, - GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH, - &offset, &remain); - if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_WA(gt, 16026007364)) - guc_waklv_enable_two_word(ads, - GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG, - 0x0, - 0xF, - &offset, &remain); + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); + if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_WA(gt, 16026007364)) { + u32 data[] = { + 0x0, + 0xF, + }; + guc_waklv_enable(ads, data, sizeof(data) / sizeof(u32), &offset, &remain, + GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG); + } size = guc_ads_waklv_size(ads) - remain; if (!size) From 90759cddaceab1a7cfd0128d9421abf2d9288d09 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:29 +0200 Subject: [PATCH 064/169] drm/xe: Simplify module initialization code There is no need to have extra checks and WARN() in the helpers as instead of an index of the entry with function pointers, we can pass pointer to the entry which we prepare directly in the main loop, that is guaranteed to be valid. add/remove: 0/0 grow/shrink: 0/4 up/down: 0/-180 (-180) Function old new delta xe_exit 109 79 -30 cleanup_module 109 79 -30 xe_init 248 188 -60 init_module 248 188 -60 Total: Before=2774145, After=2773965, chg -0.01% Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Reviewed-by: Rodrigo Vivi Reviewed-by: John Harrison Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-2-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_module.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index d9391bd08194..cfed36361613 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -135,24 +135,17 @@ static const struct init_funcs init_funcs[] = { }, }; -static int __init xe_call_init_func(unsigned int i) +static int __init xe_call_init_func(const struct init_funcs *func) { - if (WARN_ON(i >= ARRAY_SIZE(init_funcs))) - return 0; - if (!init_funcs[i].init) - return 0; - - return init_funcs[i].init(); + if (func->init) + return func->init(); + return 0; } -static void xe_call_exit_func(unsigned int i) +static void xe_call_exit_func(const struct init_funcs *func) { - if (WARN_ON(i >= ARRAY_SIZE(init_funcs))) - return; - if (!init_funcs[i].exit) - return; - - init_funcs[i].exit(); + if (func->exit) + func->exit(); } static int __init xe_init(void) @@ -160,10 +153,10 @@ static int __init xe_init(void) int err, i; for (i = 0; i < ARRAY_SIZE(init_funcs); i++) { - err = xe_call_init_func(i); + err = xe_call_init_func(init_funcs + i); if (err) { while (i--) - xe_call_exit_func(i); + xe_call_exit_func(init_funcs + i); return err; } } @@ -176,7 +169,7 @@ static void __exit xe_exit(void) int i; for (i = ARRAY_SIZE(init_funcs) - 1; i >= 0; i--) - xe_call_exit_func(i); + xe_call_exit_func(init_funcs + i); } module_init(xe_init); From 823301c847bd9c57dd8f7c684046778281744c77 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:30 +0200 Subject: [PATCH 065/169] drm/xe: Print module init abort code We should provide a hint to the user why the module refused to load. This will also allow us to drop individual error messages from init steps. Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-3-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index cfed36361613..d08338fc3bc1 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -155,6 +155,8 @@ static int __init xe_init(void) for (i = 0; i < ARRAY_SIZE(init_funcs); i++) { err = xe_call_init_func(init_funcs + i); if (err) { + pr_info("%s: module_init aborted at %ps %pe\n", + DRIVER_NAME, init_funcs[i].init, ERR_PTR(err)); while (i--) xe_call_exit_func(init_funcs + i); return err; From b90613fb02179a01c6042d8a952e2c68e81d3cf7 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:31 +0200 Subject: [PATCH 066/169] drm/xe/configfs: Destroy xe_configfs.su_mutex on exit/error While mutex_destroy() is NOP when CONFIG_DEBUG_MUTEXES is not enabled, we should still call it. While around, drop a trailing line. Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-4-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 36e2b45b305f..56fbf4c1c37f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -401,6 +401,7 @@ int __init xe_configfs_init(void) if (ret) { pr_err("Error %d while registering %s subsystem\n", ret, root->cg_item.ci_namebuf); + mutex_destroy(&xe_configfs.su_mutex); return ret; } @@ -410,5 +411,5 @@ int __init xe_configfs_init(void) void __exit xe_configfs_exit(void) { configfs_unregister_subsystem(&xe_configfs); + mutex_destroy(&xe_configfs.su_mutex); } - From c4b1dde0630e40fa773b1613495d4df4955f44b4 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:32 +0200 Subject: [PATCH 067/169] drm/xe/configfs: Drop redundant init() error message There is no need to print separate error message since we will also print one in xe_init(). Also drop temporary variable, which was likely just taken from the example code. Signed-off-by: Michal Wajdeczko Reviewed-by: John Harrison Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-5-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 56fbf4c1c37f..6aa0531bcf76 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -392,15 +392,12 @@ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) int __init xe_configfs_init(void) { - struct config_group *root = &xe_configfs.su_group; int ret; - config_group_init(root); + config_group_init(&xe_configfs.su_group); mutex_init(&xe_configfs.su_mutex); ret = configfs_register_subsystem(&xe_configfs); if (ret) { - pr_err("Error %d while registering %s subsystem\n", - ret, root->cg_item.ci_namebuf); mutex_destroy(&xe_configfs.su_mutex); return ret; } From 88df7939d728a90ddb722c8113a4529948b3d2ef Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:33 +0200 Subject: [PATCH 068/169] drm/xe/configfs: Rename struct xe_config_device Rename it to struct xe_config_group_device to better match its purpose. It will also help us to reintroduce in the upcoming patch the same struct name but this time to hold only configuration data. Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-6-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 6aa0531bcf76..d69b5f248e59 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -85,7 +85,7 @@ * rmdir /sys/kernel/config/xe/0000:03:00.0/ */ -struct xe_config_device { +struct xe_config_group_device { struct config_group group; bool survivability_mode; @@ -113,21 +113,21 @@ static const struct engine_info engine_info[] = { { .cls = "gsccs", .mask = XE_HW_ENGINE_GSCCS_MASK }, }; -static struct xe_config_device *to_xe_config_device(struct config_item *item) +static struct xe_config_group_device *to_xe_config_group_device(struct config_item *item) { - return container_of(to_config_group(item), struct xe_config_device, group); + return container_of(to_config_group(item), struct xe_config_group_device, group); } static ssize_t survivability_mode_show(struct config_item *item, char *page) { - struct xe_config_device *dev = to_xe_config_device(item); + struct xe_config_group_device *dev = to_xe_config_group_device(item); return sprintf(page, "%d\n", dev->survivability_mode); } static ssize_t survivability_mode_store(struct config_item *item, const char *page, size_t len) { - struct xe_config_device *dev = to_xe_config_device(item); + struct xe_config_group_device *dev = to_xe_config_group_device(item); bool survivability_mode; int ret; @@ -144,7 +144,7 @@ static ssize_t survivability_mode_store(struct config_item *item, const char *pa static ssize_t engines_allowed_show(struct config_item *item, char *page) { - struct xe_config_device *dev = to_xe_config_device(item); + struct xe_config_group_device *dev = to_xe_config_group_device(item); char *p = page; for (size_t i = 0; i < ARRAY_SIZE(engine_info); i++) { @@ -199,7 +199,7 @@ static bool lookup_engine_mask(const char *pattern, u64 *mask) static ssize_t engines_allowed_store(struct config_item *item, const char *page, size_t len) { - struct xe_config_device *dev = to_xe_config_device(item); + struct xe_config_group_device *dev = to_xe_config_group_device(item); size_t patternlen, p; u64 mask, val = 0; @@ -237,7 +237,7 @@ static struct configfs_attribute *xe_config_device_attrs[] = { static void xe_config_device_release(struct config_item *item) { - struct xe_config_device *dev = to_xe_config_device(item); + struct xe_config_group_device *dev = to_xe_config_group_device(item); mutex_destroy(&dev->lock); kfree(dev); @@ -257,7 +257,7 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro const char *name) { unsigned int domain, bus, slot, function; - struct xe_config_device *dev; + struct xe_config_group_device *dev; struct pci_dev *pdev; char canonical[16]; int ret; @@ -309,7 +309,7 @@ static struct configfs_subsystem xe_configfs = { }, }; -static struct xe_config_device *configfs_find_group(struct pci_dev *pdev) +static struct xe_config_group_device *configfs_find_group(struct pci_dev *pdev) { struct config_item *item; @@ -320,7 +320,7 @@ static struct xe_config_device *configfs_find_group(struct pci_dev *pdev) if (!item) return NULL; - return to_xe_config_device(item); + return to_xe_config_group_device(item); } /** @@ -334,7 +334,7 @@ static struct xe_config_device *configfs_find_group(struct pci_dev *pdev) */ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { - struct xe_config_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = configfs_find_group(pdev); bool mode; if (!dev) @@ -355,7 +355,7 @@ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) */ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) { - struct xe_config_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = configfs_find_group(pdev); if (!dev) return; @@ -378,7 +378,7 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) */ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { - struct xe_config_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = configfs_find_group(pdev); u64 engines_allowed; if (!dev) From ae3184d5f980d92377545ba58030e47ce48dd09f Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:34 +0200 Subject: [PATCH 069/169] drm/xe/configfs: Rename configfs_find_group() helper This helper name shouldn't suggest that it iss a part of the core configfs API family. While around switch to use different helper to release a reference. Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-7-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index d69b5f248e59..acdee616d118 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -309,7 +309,7 @@ static struct configfs_subsystem xe_configfs = { }, }; -static struct xe_config_group_device *configfs_find_group(struct pci_dev *pdev) +static struct xe_config_group_device *find_xe_config_group_device(struct pci_dev *pdev) { struct config_item *item; @@ -334,14 +334,14 @@ static struct xe_config_group_device *configfs_find_group(struct pci_dev *pdev) */ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { - struct xe_config_group_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); bool mode; if (!dev) return false; mode = dev->survivability_mode; - config_item_put(&dev->group.cg_item); + config_group_put(&dev->group); return mode; } @@ -355,7 +355,7 @@ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) */ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) { - struct xe_config_group_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); if (!dev) return; @@ -364,7 +364,7 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) dev->survivability_mode = 0; mutex_unlock(&dev->lock); - config_item_put(&dev->group.cg_item); + config_group_put(&dev->group); } /** @@ -378,14 +378,14 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) */ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { - struct xe_config_group_device *dev = configfs_find_group(pdev); + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); u64 engines_allowed; if (!dev) return U64_MAX; engines_allowed = dev->engines_allowed; - config_item_put(&dev->group.cg_item); + config_group_put(&dev->group); return engines_allowed; } From 3c643f6216215c7e4e30f7a647024a20847b27af Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:35 +0200 Subject: [PATCH 070/169] drm/xe/configfs: Reintroduce struct xe_config_device This time it will hold just pure configuration parameters, without any configfs related stuff. This will help us define defaults data without wasting space for unneeded data. Signed-off-by: Michal Wajdeczko Reviewed-by: John Harrison Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-8-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index acdee616d118..7ad9fc65e21d 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -88,8 +88,10 @@ struct xe_config_group_device { struct config_group group; - bool survivability_mode; - u64 engines_allowed; + struct xe_config_device { + bool survivability_mode; + u64 engines_allowed; + } config; /* protects attributes */ struct mutex lock; @@ -118,9 +120,14 @@ static struct xe_config_group_device *to_xe_config_group_device(struct config_it return container_of(to_config_group(item), struct xe_config_group_device, group); } +static struct xe_config_device *to_xe_config_device(struct config_item *item) +{ + return &to_xe_config_group_device(item)->config; +} + static ssize_t survivability_mode_show(struct config_item *item, char *page) { - struct xe_config_group_device *dev = to_xe_config_group_device(item); + struct xe_config_device *dev = to_xe_config_device(item); return sprintf(page, "%d\n", dev->survivability_mode); } @@ -136,7 +143,7 @@ static ssize_t survivability_mode_store(struct config_item *item, const char *pa return ret; mutex_lock(&dev->lock); - dev->survivability_mode = survivability_mode; + dev->config.survivability_mode = survivability_mode; mutex_unlock(&dev->lock); return len; @@ -144,7 +151,7 @@ static ssize_t survivability_mode_store(struct config_item *item, const char *pa static ssize_t engines_allowed_show(struct config_item *item, char *page) { - struct xe_config_group_device *dev = to_xe_config_group_device(item); + struct xe_config_device *dev = to_xe_config_device(item); char *p = page; for (size_t i = 0; i < ARRAY_SIZE(engine_info); i++) { @@ -220,7 +227,7 @@ static ssize_t engines_allowed_store(struct config_item *item, const char *page, } mutex_lock(&dev->lock); - dev->engines_allowed = val; + dev->config.engines_allowed = val; mutex_unlock(&dev->lock); return len; @@ -282,7 +289,7 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro return ERR_PTR(-ENOMEM); /* Default values */ - dev->engines_allowed = U64_MAX; + dev->config.engines_allowed = U64_MAX; config_group_init_type_name(&dev->group, name, &xe_config_device_type); @@ -340,7 +347,7 @@ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) if (!dev) return false; - mode = dev->survivability_mode; + mode = dev->config.survivability_mode; config_group_put(&dev->group); return mode; @@ -361,7 +368,7 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) return; mutex_lock(&dev->lock); - dev->survivability_mode = 0; + dev->config.survivability_mode = 0; mutex_unlock(&dev->lock); config_group_put(&dev->group); @@ -384,7 +391,7 @@ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) if (!dev) return U64_MAX; - engines_allowed = dev->engines_allowed; + engines_allowed = dev->config.engines_allowed; config_group_put(&dev->group); return engines_allowed; From 737a72d7e0231123ae3f1578845d9a23b6d02a60 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:36 +0200 Subject: [PATCH 071/169] drm/xe/configfs: Keep default device config settings together For easier maintenance add a placeholder where we can keep all default device configuration settings in one place. Signed-off-by: Michal Wajdeczko Reviewed-by: John Harrison Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-9-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 7ad9fc65e21d..3fc01bef0114 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -97,6 +97,16 @@ struct xe_config_group_device { struct mutex lock; }; +static const struct xe_config_device device_defaults = { + .survivability_mode = false, + .engines_allowed = U64_MAX, +}; + +static void set_device_defaults(struct xe_config_device *config) +{ + *config = device_defaults; +} + struct engine_info { const char *cls; u64 mask; @@ -288,8 +298,7 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro if (!dev) return ERR_PTR(-ENOMEM); - /* Default values */ - dev->config.engines_allowed = U64_MAX; + set_device_defaults(&dev->config); config_group_init_type_name(&dev->group, name, &xe_config_device_type); @@ -345,7 +354,7 @@ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) bool mode; if (!dev) - return false; + return device_defaults.survivability_mode; mode = dev->config.survivability_mode; config_group_put(&dev->group); @@ -389,7 +398,7 @@ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) u64 engines_allowed; if (!dev) - return U64_MAX; + return device_defaults.engines_allowed; engines_allowed = dev->config.engines_allowed; config_group_put(&dev->group); From b4687422c322e5afc0f5d9425b29080c06dc17b9 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 21:33:38 +0200 Subject: [PATCH 072/169] drm/xe/configfs: Only allow configurations for supported devices Since we already lookup for the real PCI device before we allow to create its directory config, we might also check if the found device matches our driver PCI ID list. This will prevent creation of the directory configs for the unsupported devices. Signed-off-by: Michal Wajdeczko Cc: John Harrison Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731193339.179829-11-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 3fc01bef0114..5c13749733af 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -270,11 +270,22 @@ static const struct config_item_type xe_config_device_type = { .ct_owner = THIS_MODULE, }; +static const struct xe_device_desc *xe_match_desc(struct pci_dev *pdev) +{ + struct device_driver *driver = driver_find("xe", &pci_bus_type); + struct pci_driver *drv = to_pci_driver(driver); + const struct pci_device_id *ids = drv ? drv->id_table : NULL; + const struct pci_device_id *found = pci_match_id(ids, pdev); + + return found ? (const void *)found->driver_data : NULL; +} + static struct config_group *xe_config_make_device_group(struct config_group *group, const char *name) { unsigned int domain, bus, slot, function; struct xe_config_group_device *dev; + const struct xe_device_desc *match; struct pci_dev *pdev; char canonical[16]; int ret; @@ -292,8 +303,16 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); if (!pdev) return ERR_PTR(-ENODEV); + + match = xe_match_desc(pdev); + if (!match) + pci_info(pdev, "xe driver does not support configuration of this device\n"); + pci_dev_put(pdev); + if (!match) + return ERR_PTR(-ENOENT); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) return ERR_PTR(-ENOMEM); From ca0ed3b10f5dba790808107896c568cc037a74f4 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Thu, 31 Jul 2025 23:21:45 +0200 Subject: [PATCH 073/169] drm/xe/configfs: Allow adding configurations for future VFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we are expecting that all configuration directory names will match some of the existing devices, we can't provide any configuration for the VFs until they are actually enabled. But we can relax that restriction by just checking if there is a PF device that could create given VF. This is easy since all our PF devices are always present at function 0 and we can query PF device for number of VFs it could support. Then for some system with PF device at 0000:00:02.0 we can add configs for all VFs: /sys/kernel/config/xe/ ├── 0000:00:02.0 │   └── ... ├── 0000:00:02.1 │   └── ... ├── 0000:00:02.2 │   └── ... : └── 0000:00:02.7 └── ... Signed-off-by: Michal Wajdeczko Cc: John Harrison Reviewed-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250731212145.179898-1-michal.wajdeczko@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 33 +++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 5c13749733af..853da2ee837a 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -12,9 +12,9 @@ #include #include "xe_configfs.h" -#include "xe_module.h" - #include "xe_hw_engine_types.h" +#include "xe_module.h" +#include "xe_pci_types.h" /** * DOC: Xe Configfs @@ -280,6 +280,15 @@ static const struct xe_device_desc *xe_match_desc(struct pci_dev *pdev) return found ? (const void *)found->driver_data : NULL; } +static struct pci_dev *get_physfn_instead(struct pci_dev *virtfn) +{ + struct pci_dev *physfn = pci_physfn(virtfn); + + pci_dev_get(physfn); + pci_dev_put(virtfn); + return physfn; +} + static struct config_group *xe_config_make_device_group(struct config_group *group, const char *name) { @@ -288,6 +297,7 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro const struct xe_device_desc *match; struct pci_dev *pdev; char canonical[16]; + int vfnumber = 0; int ret; ret = sscanf(name, "%x:%x:%x.%x", &domain, &bus, &slot, &function); @@ -301,12 +311,29 @@ static struct config_group *xe_config_make_device_group(struct config_group *gro return ERR_PTR(-EINVAL); pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); + if (!pdev && function) + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, 0)); + if (!pdev && slot) + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(0, 0)); if (!pdev) return ERR_PTR(-ENODEV); + if (PCI_DEVFN(slot, function) != pdev->devfn) { + pdev = get_physfn_instead(pdev); + vfnumber = PCI_DEVFN(slot, function) - pdev->devfn; + if (!dev_is_pf(&pdev->dev) || vfnumber > pci_sriov_get_totalvfs(pdev)) { + pci_dev_put(pdev); + return ERR_PTR(-ENODEV); + } + } + match = xe_match_desc(pdev); - if (!match) + if (match && vfnumber && !match->has_sriov) { + pci_info(pdev, "xe driver does not support VFs on this device\n"); + match = NULL; + } else if (!match) { pci_info(pdev, "xe driver does not support configuration of this device\n"); + } pci_dev_put(pdev); From 81aa3c7c62049b42959bd4054c3a3ed34b2d5bf4 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:02 +0200 Subject: [PATCH 074/169] drm/pagemap: Rename drm_pagemap_device_addr to drm_pagemap_addr Rename this struct to the more generic name drm_pagemap_addr so it can be used in a broader context, such as DMA mappings of CPU memory. Reviewed-by: Matthew Brost Acked-by: Maarten Lankhorst Link: https://lore.kernel.org/r/20250805140028.599361-2-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/drm_gpusvm.c | 4 ++-- drivers/gpu/drm/xe/xe_res_cursor.h | 10 +++++----- drivers/gpu/drm/xe/xe_svm.c | 4 ++-- drivers/gpu/drm/xe/xe_svm.h | 4 ++-- include/drm/drm_gpusvm.h | 4 ++-- include/drm/drm_pagemap.h | 32 +++++++++++++++--------------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 5bb4c77db2c3..661306da6b2d 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1048,7 +1048,7 @@ static void __drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, }; for (i = 0, j = 0; i < npages; j++) { - struct drm_pagemap_device_addr *addr = &range->dma_addr[j]; + struct drm_pagemap_addr *addr = &range->dma_addr[j]; if (addr->proto == DRM_INTERCONNECT_SYSTEM) dma_unmap_page(dev, @@ -1400,7 +1400,7 @@ map_pages: goto err_unmap; } - range->dma_addr[j] = drm_pagemap_device_addr_encode + range->dma_addr[j] = drm_pagemap_addr_encode (addr, DRM_INTERCONNECT_SYSTEM, order, DMA_BIDIRECTIONAL); } diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index d1a403cfb628..4e00008b7081 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -55,8 +55,8 @@ struct xe_res_cursor { u32 mem_type; /** @sgl: Scatterlist for cursor */ struct scatterlist *sgl; - /** @dma_addr: Current element in a struct drm_pagemap_device_addr array */ - const struct drm_pagemap_device_addr *dma_addr; + /** @dma_addr: Current element in a struct drm_pagemap_addr array */ + const struct drm_pagemap_addr *dma_addr; /** @mm: Buddy allocator for VRAM cursor */ struct drm_buddy *mm; /** @@ -170,7 +170,7 @@ static inline void __xe_res_sg_next(struct xe_res_cursor *cur) */ static inline void __xe_res_dma_next(struct xe_res_cursor *cur) { - const struct drm_pagemap_device_addr *addr = cur->dma_addr; + const struct drm_pagemap_addr *addr = cur->dma_addr; u64 start = cur->start; while (start >= cur->dma_seg_size) { @@ -222,14 +222,14 @@ static inline void xe_res_first_sg(const struct sg_table *sg, /** * xe_res_first_dma - initialize a xe_res_cursor with dma_addr array * - * @dma_addr: struct drm_pagemap_device_addr array to walk + * @dma_addr: struct drm_pagemap_addr array to walk * @start: Start of the range * @size: Size of the range * @cur: cursor object to initialize * * Start walking over the range of allocations between @start and @size. */ -static inline void xe_res_first_dma(const struct drm_pagemap_device_addr *dma_addr, +static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, u64 start, u64 size, struct xe_res_cursor *cur) { diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 10c8a1bcb86e..3e4f4a8760ee 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -1014,7 +1014,7 @@ int xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, ctx->timeslice_ms); } -static struct drm_pagemap_device_addr +static struct drm_pagemap_addr xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap, struct device *dev, struct page *page, @@ -1033,7 +1033,7 @@ xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap, prot = 0; } - return drm_pagemap_device_addr_encode(addr, prot, order, dir); + return drm_pagemap_addr_encode(addr, prot, order, dir); } static const struct drm_pagemap_ops xe_drm_pagemap_ops = { diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index da9a69ea0bb1..4bdccb56d25f 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -163,7 +163,7 @@ void xe_svm_flush(struct xe_vm *vm); #else #include -struct drm_pagemap_device_addr; +struct drm_pagemap_addr; struct drm_gpusvm_ctx; struct drm_gpusvm_range; struct xe_bo; @@ -178,7 +178,7 @@ struct xe_vram_region; struct xe_svm_range { struct { struct interval_tree_node itree; - const struct drm_pagemap_device_addr *dma_addr; + const struct drm_pagemap_addr *dma_addr; } base; u32 tile_present; u32 tile_invalidated; diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index 4aedc5423aff..8d613e9b2690 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -17,7 +17,7 @@ struct drm_gpusvm_notifier; struct drm_gpusvm_ops; struct drm_gpusvm_range; struct drm_pagemap; -struct drm_pagemap_device_addr; +struct drm_pagemap_addr; /** * struct drm_gpusvm_ops - Operations structure for GPU SVM @@ -154,7 +154,7 @@ struct drm_gpusvm_range { struct interval_tree_node itree; struct list_head entry; unsigned long notifier_seq; - struct drm_pagemap_device_addr *dma_addr; + struct drm_pagemap_addr *dma_addr; struct drm_pagemap *dpagemap; struct drm_gpusvm_range_flags flags; }; diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index e5f20a1235be..69d6ee49a3de 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -23,7 +23,7 @@ enum drm_interconnect_protocol { }; /** - * struct drm_pagemap_device_addr - Device address representation. + * struct drm_pagemap_addr - Address representation. * @addr: The dma address or driver-defined address for driver private interconnects. * @proto: The interconnect protocol. * @order: The page order of the device mapping. (Size is PAGE_SIZE << order). @@ -32,7 +32,7 @@ enum drm_interconnect_protocol { * Note: There is room for improvement here. We should be able to pack into * 64 bits. */ -struct drm_pagemap_device_addr { +struct drm_pagemap_addr { dma_addr_t addr; u64 proto : 54; u64 order : 8; @@ -40,21 +40,21 @@ struct drm_pagemap_device_addr { }; /** - * drm_pagemap_device_addr_encode() - Encode a dma address with metadata + * drm_pagemap_addr_encode() - Encode a dma address with metadata * @addr: The dma address or driver-defined address for driver private interconnects. * @proto: The interconnect protocol. * @order: The page order of the dma mapping. (Size is PAGE_SIZE << order). * @dir: The DMA direction. * - * Return: A struct drm_pagemap_device_addr encoding the above information. + * Return: A struct drm_pagemap_addr encoding the above information. */ -static inline struct drm_pagemap_device_addr -drm_pagemap_device_addr_encode(dma_addr_t addr, - enum drm_interconnect_protocol proto, - unsigned int order, - enum dma_data_direction dir) +static inline struct drm_pagemap_addr +drm_pagemap_addr_encode(dma_addr_t addr, + enum drm_interconnect_protocol proto, + unsigned int order, + enum dma_data_direction dir) { - return (struct drm_pagemap_device_addr) { + return (struct drm_pagemap_addr) { .addr = addr, .proto = proto, .order = order, @@ -75,11 +75,11 @@ struct drm_pagemap_ops { * @order: The page order of the device mapping. (Size is PAGE_SIZE << order). * @dir: The transfer direction. */ - struct drm_pagemap_device_addr (*device_map)(struct drm_pagemap *dpagemap, - struct device *dev, - struct page *page, - unsigned int order, - enum dma_data_direction dir); + struct drm_pagemap_addr (*device_map)(struct drm_pagemap *dpagemap, + struct device *dev, + struct page *page, + unsigned int order, + enum dma_data_direction dir); /** * @device_unmap: Unmap a device address previously obtained using @device_map. @@ -90,7 +90,7 @@ struct drm_pagemap_ops { */ void (*device_unmap)(struct drm_pagemap *dpagemap, struct device *dev, - struct drm_pagemap_device_addr addr); + struct drm_pagemap_addr addr); /** * @populate_mm: Populate part of the mm with @dpagemap memory, From f35a6cdf8a6d69f2fb35ece202a09f13fe7c87b2 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:03 +0200 Subject: [PATCH 075/169] drm/pagemap: Use struct drm_pagemap_addr in mapping and copy functions This struct embeds more information than just the DMA address. This will help later to support folio orders greater than zero. At this point, there is no functional change as the only struct member used is addr. In Xe, adapt to the new drm_gpusvm_devmem_ops type signatures using struct drm_pagemap_addr, as well as the internal xe SVM functions implementing those operations. The use of this struct is propagated to xe_migrate as it makes indexed accesses to the next DMA address but they are no longer contiguous. v2: - Rename drm_pagemap_device_addr to drm_pagemap_addr (Matthew Brost) - Squash with patch for Xe (Matthew Brost) - Set proto and dir for completeness (Matthew Brost) - Assess DMA map protocol (Matthew Brost) Cc: Matthew Brost Reviewed-by: Matthew Brost Acked-by: Maarten Lankhorst Link: https://lore.kernel.org/r/20250805140028.599361-3-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/drm_pagemap.c | 65 +++++++++++++++++-------------- drivers/gpu/drm/xe/xe_migrate.c | 69 +++++++++++++++++++-------------- drivers/gpu/drm/xe/xe_migrate.h | 5 ++- drivers/gpu/drm/xe/xe_svm.c | 37 ++++++++++-------- include/drm/drm_pagemap.h | 8 ++-- 5 files changed, 104 insertions(+), 80 deletions(-) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 1da55322af12..e5c135d793ed 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -202,7 +202,7 @@ static void drm_pagemap_get_devmem_page(struct page *page, /** * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration * @dev: The device for which the pages are being mapped - * @dma_addr: Array to store DMA addresses corresponding to mapped pages + * @pagemap_addr: Array to store DMA information corresponding to mapped pages * @migrate_pfn: Array of migrate page frame numbers to map * @npages: Number of pages to map * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) @@ -215,7 +215,7 @@ static void drm_pagemap_get_devmem_page(struct page *page, * Returns: 0 on success, -EFAULT if an error occurs during mapping. */ static int drm_pagemap_migrate_map_pages(struct device *dev, - dma_addr_t *dma_addr, + struct drm_pagemap_addr *pagemap_addr, unsigned long *migrate_pfn, unsigned long npages, enum dma_data_direction dir) @@ -224,6 +224,7 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, for (i = 0; i < npages; ++i) { struct page *page = migrate_pfn_to_page(migrate_pfn[i]); + dma_addr_t dma_addr; if (!page) continue; @@ -231,9 +232,14 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, if (WARN_ON_ONCE(is_zone_device_page(page))) return -EFAULT; - dma_addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir); - if (dma_mapping_error(dev, dma_addr[i])) + dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, dir); + if (dma_mapping_error(dev, dma_addr)) return -EFAULT; + + pagemap_addr[i] = + drm_pagemap_addr_encode(dma_addr, + DRM_INTERCONNECT_SYSTEM, + 0, dir); } return 0; @@ -242,7 +248,7 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, /** * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration * @dev: The device for which the pages were mapped - * @dma_addr: Array of DMA addresses corresponding to mapped pages + * @pagemap_addr: Array of DMA information corresponding to mapped pages * @npages: Number of pages to unmap * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) * @@ -251,17 +257,17 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, * if it's valid and not already unmapped, and unmaps the corresponding page. */ static void drm_pagemap_migrate_unmap_pages(struct device *dev, - dma_addr_t *dma_addr, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages, enum dma_data_direction dir) { unsigned long i; for (i = 0; i < npages; ++i) { - if (!dma_addr[i] || dma_mapping_error(dev, dma_addr[i])) + if (!pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) continue; - dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir); + dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE, dir); } } @@ -314,7 +320,7 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, struct vm_area_struct *vas; struct drm_pagemap_zdd *zdd = NULL; struct page **pages; - dma_addr_t *dma_addr; + struct drm_pagemap_addr *pagemap_addr; void *buf; int err; @@ -340,14 +346,14 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, goto err_out; } - buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*dma_addr) + + buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) + sizeof(*pages), GFP_KERNEL); if (!buf) { err = -ENOMEM; goto err_out; } - dma_addr = buf + (2 * sizeof(*migrate.src) * npages); - pages = buf + (2 * sizeof(*migrate.src) + sizeof(*dma_addr)) * npages; + pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); + pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; zdd = drm_pagemap_zdd_alloc(pgmap_owner); if (!zdd) { @@ -377,8 +383,9 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, if (err) goto err_finalize; - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, dma_addr, + err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, migrate.src, npages, DMA_TO_DEVICE); + if (err) goto err_finalize; @@ -390,7 +397,7 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, drm_pagemap_get_devmem_page(page, zdd); } - err = ops->copy_to_devmem(pages, dma_addr, npages); + err = ops->copy_to_devmem(pages, pagemap_addr, npages); if (err) goto err_finalize; @@ -404,7 +411,7 @@ err_finalize: drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); migrate_vma_pages(&migrate); migrate_vma_finalize(&migrate); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, dma_addr, npages, + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, DMA_TO_DEVICE); err_free: if (zdd) @@ -509,7 +516,7 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) unsigned long npages, mpages = 0; struct page **pages; unsigned long *src, *dst; - dma_addr_t *dma_addr; + struct drm_pagemap_addr *pagemap_addr; void *buf; int i, err = 0; unsigned int retry_count = 2; @@ -520,7 +527,7 @@ retry: if (!mmget_not_zero(devmem_allocation->mm)) return -EFAULT; - buf = kvcalloc(npages, 2 * sizeof(*src) + sizeof(*dma_addr) + + buf = kvcalloc(npages, 2 * sizeof(*src) + sizeof(*pagemap_addr) + sizeof(*pages), GFP_KERNEL); if (!buf) { err = -ENOMEM; @@ -528,8 +535,8 @@ retry: } src = buf; dst = buf + (sizeof(*src) * npages); - dma_addr = buf + (2 * sizeof(*src) * npages); - pages = buf + (2 * sizeof(*src) + sizeof(*dma_addr)) * npages; + pagemap_addr = buf + (2 * sizeof(*src) * npages); + pages = buf + (2 * sizeof(*src) + sizeof(*pagemap_addr)) * npages; err = ops->populate_devmem_pfn(devmem_allocation, npages, src); if (err) @@ -544,7 +551,7 @@ retry: if (err || !mpages) goto err_finalize; - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, dma_addr, + err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, dst, npages, DMA_FROM_DEVICE); if (err) goto err_finalize; @@ -552,7 +559,7 @@ retry: for (i = 0; i < npages; ++i) pages[i] = migrate_pfn_to_page(src[i]); - err = ops->copy_to_ram(pages, dma_addr, npages); + err = ops->copy_to_ram(pages, pagemap_addr, npages); if (err) goto err_finalize; @@ -561,7 +568,7 @@ err_finalize: drm_pagemap_migration_unlock_put_pages(npages, dst); migrate_device_pages(src, dst, npages); migrate_device_finalize(src, dst, npages); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, dma_addr, npages, + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, DMA_FROM_DEVICE); err_free: kvfree(buf); @@ -612,7 +619,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, struct device *dev = NULL; unsigned long npages, mpages = 0; struct page **pages; - dma_addr_t *dma_addr; + struct drm_pagemap_addr *pagemap_addr; unsigned long start, end; void *buf; int i, err = 0; @@ -637,14 +644,14 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, migrate.end = end; npages = npages_in_range(start, end); - buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*dma_addr) + + buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) + sizeof(*pages), GFP_KERNEL); if (!buf) { err = -ENOMEM; goto err_out; } - dma_addr = buf + (2 * sizeof(*migrate.src) * npages); - pages = buf + (2 * sizeof(*migrate.src) + sizeof(*dma_addr)) * npages; + pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); + pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; migrate.vma = vas; migrate.src = buf; @@ -680,7 +687,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (err) goto err_finalize; - err = drm_pagemap_migrate_map_pages(dev, dma_addr, migrate.dst, npages, + err = drm_pagemap_migrate_map_pages(dev, pagemap_addr, migrate.dst, npages, DMA_FROM_DEVICE); if (err) goto err_finalize; @@ -688,7 +695,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, for (i = 0; i < npages; ++i) pages[i] = migrate_pfn_to_page(migrate.src[i]); - err = ops->copy_to_ram(pages, dma_addr, npages); + err = ops->copy_to_ram(pages, pagemap_addr, npages); if (err) goto err_finalize; @@ -698,7 +705,7 @@ err_finalize: migrate_vma_pages(&migrate); migrate_vma_finalize(&migrate); if (dev) - drm_pagemap_migrate_unmap_pages(dev, dma_addr, npages, + drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages, DMA_FROM_DEVICE); err_free: kvfree(buf); diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 3a276e2348a2..6a0f04411ae0 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -1738,7 +1739,8 @@ static u32 pte_update_cmd_size(u64 size) static void build_pt_update_batch_sram(struct xe_migrate *m, struct xe_bb *bb, u32 pt_offset, - dma_addr_t *sram_addr, u32 size) + struct drm_pagemap_addr *sram_addr, + u32 size) { u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB]; u32 ptes; @@ -1756,14 +1758,18 @@ static void build_pt_update_batch_sram(struct xe_migrate *m, ptes -= chunk; while (chunk--) { - u64 addr = sram_addr[i++] & PAGE_MASK; + u64 addr = sram_addr[i].addr & PAGE_MASK; + xe_tile_assert(m->tile, sram_addr[i].proto == + DRM_INTERCONNECT_SYSTEM); xe_tile_assert(m->tile, addr); addr = m->q->vm->pt_ops->pte_encode_addr(m->tile->xe, addr, pat_index, 0, false, 0); bb->cs[bb->len++] = lower_32_bits(addr); bb->cs[bb->len++] = upper_32_bits(addr); + + i++; } } } @@ -1779,7 +1785,8 @@ enum xe_migrate_copy_dir { static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, unsigned long len, unsigned long sram_offset, - dma_addr_t *sram_addr, u64 vram_addr, + struct drm_pagemap_addr *sram_addr, + u64 vram_addr, const enum xe_migrate_copy_dir dir) { struct xe_gt *gt = m->tile->primary_gt; @@ -1861,7 +1868,7 @@ err: * xe_migrate_to_vram() - Migrate to VRAM * @m: The migration context. * @npages: Number of pages to migrate. - * @src_addr: Array of dma addresses (source of migrate) + * @src_addr: Array of DMA information (source of migrate) * @dst_addr: Device physical address of VRAM (destination of migrate) * * Copy from an array dma addresses to a VRAM device physical address @@ -1871,7 +1878,7 @@ err: */ struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, unsigned long npages, - dma_addr_t *src_addr, + struct drm_pagemap_addr *src_addr, u64 dst_addr) { return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, @@ -1883,7 +1890,7 @@ struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, * @m: The migration context. * @npages: Number of pages to migrate. * @src_addr: Device physical address of VRAM (source of migrate) - * @dst_addr: Array of dma addresses (destination of migrate) + * @dst_addr: Array of DMA information (destination of migrate) * * Copy from a VRAM device physical address to an array dma addresses * @@ -1893,61 +1900,65 @@ struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, unsigned long npages, u64 src_addr, - dma_addr_t *dst_addr) + struct drm_pagemap_addr *dst_addr) { return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, XE_MIGRATE_COPY_TO_SRAM); } -static void xe_migrate_dma_unmap(struct xe_device *xe, dma_addr_t *dma_addr, +static void xe_migrate_dma_unmap(struct xe_device *xe, + struct drm_pagemap_addr *pagemap_addr, int len, int write) { unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); for (i = 0; i < npages; ++i) { - if (!dma_addr[i]) + if (!pagemap_addr[i].addr) break; - dma_unmap_page(xe->drm.dev, dma_addr[i], PAGE_SIZE, + dma_unmap_page(xe->drm.dev, pagemap_addr[i].addr, PAGE_SIZE, write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); } - kfree(dma_addr); + kfree(pagemap_addr); } -static dma_addr_t *xe_migrate_dma_map(struct xe_device *xe, - void *buf, int len, int write) +static struct drm_pagemap_addr *xe_migrate_dma_map(struct xe_device *xe, + void *buf, int len, + int write) { - dma_addr_t *dma_addr; + struct drm_pagemap_addr *pagemap_addr; unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); - dma_addr = kcalloc(npages, sizeof(*dma_addr), GFP_KERNEL); - if (!dma_addr) + pagemap_addr = kcalloc(npages, sizeof(*pagemap_addr), GFP_KERNEL); + if (!pagemap_addr) return ERR_PTR(-ENOMEM); for (i = 0; i < npages; ++i) { dma_addr_t addr; struct page *page; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : + DMA_FROM_DEVICE; if (is_vmalloc_addr(buf)) page = vmalloc_to_page(buf); else page = virt_to_page(buf); - addr = dma_map_page(xe->drm.dev, - page, 0, PAGE_SIZE, - write ? DMA_TO_DEVICE : - DMA_FROM_DEVICE); + addr = dma_map_page(xe->drm.dev, page, 0, PAGE_SIZE, dir); if (dma_mapping_error(xe->drm.dev, addr)) goto err_fault; - dma_addr[i] = addr; + pagemap_addr[i] = + drm_pagemap_addr_encode(addr, + DRM_INTERCONNECT_SYSTEM, + 0, dir); buf += PAGE_SIZE; } - return dma_addr; + return pagemap_addr; err_fault: - xe_migrate_dma_unmap(xe, dma_addr, len, write); + xe_migrate_dma_unmap(xe, pagemap_addr, len, write); return ERR_PTR(-EFAULT); } @@ -1976,7 +1987,7 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, struct xe_device *xe = tile_to_xe(tile); struct xe_res_cursor cursor; struct dma_fence *fence = NULL; - dma_addr_t *dma_addr; + struct drm_pagemap_addr *pagemap_addr; unsigned long page_offset = (unsigned long)buf & ~PAGE_MASK; int bytes_left = len, current_page = 0; void *orig_buf = buf; @@ -2031,9 +2042,9 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, return 0; } - dma_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write); - if (IS_ERR(dma_addr)) - return PTR_ERR(dma_addr); + pagemap_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write); + if (IS_ERR(pagemap_addr)) + return PTR_ERR(pagemap_addr); xe_res_first(bo->ttm.resource, offset, xe_bo_size(bo) - offset, &cursor); @@ -2054,7 +2065,7 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, __fence = xe_migrate_vram(m, current_bytes, (unsigned long)buf & ~PAGE_MASK, - dma_addr + current_page, + &pagemap_addr[current_page], vram_addr, write ? XE_MIGRATE_COPY_TO_VRAM : XE_MIGRATE_COPY_TO_SRAM); @@ -2078,7 +2089,7 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, dma_fence_put(fence); out_err: - xe_migrate_dma_unmap(xe, dma_addr, len + page_offset, write); + xe_migrate_dma_unmap(xe, pagemap_addr, len + page_offset, write); return IS_ERR(fence) ? PTR_ERR(fence) : 0; } diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index e81ea6b27fb5..a3f3fa281e04 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -9,6 +9,7 @@ #include struct dma_fence; +struct drm_pagemap_addr; struct iosys_map; struct ttm_resource; @@ -109,13 +110,13 @@ int xe_migrate_init(struct xe_migrate *m); struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, unsigned long npages, - dma_addr_t *src_addr, + struct drm_pagemap_addr *src_addr, u64 dst_addr); struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, unsigned long npages, u64 src_addr, - dma_addr_t *dst_addr); + struct drm_pagemap_addr *dst_addr); struct dma_fence *xe_migrate_copy(struct xe_migrate *m, struct xe_bo *src_bo, diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 3e4f4a8760ee..1d097e76aabc 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -328,7 +328,8 @@ enum xe_svm_copy_dir { XE_SVM_COPY_TO_SRAM, }; -static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, +static int xe_svm_copy(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages, const enum xe_svm_copy_dir dir) { struct xe_vram_region *vr = NULL; @@ -360,7 +361,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, last = (i + 1) == npages; /* No CPU page and no device pages queue'd to copy */ - if (!dma_addr[i] && vram_addr == XE_VRAM_ADDR_INVALID) + if (!pagemap_addr[i].addr && vram_addr == XE_VRAM_ADDR_INVALID) continue; if (!vr && spage) { @@ -374,7 +375,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, * first device page, check if physical contiguous on subsequent * device pages. */ - if (dma_addr[i] && spage) { + if (pagemap_addr[i].addr && spage) { __vram_addr = xe_vram_region_page_to_dpa(vr, spage); if (vram_addr == XE_VRAM_ADDR_INVALID) { vram_addr = __vram_addr; @@ -399,18 +400,20 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, if (sram) { vm_dbg(&xe->drm, "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", - vram_addr, (u64)dma_addr[pos], i - pos + incr); + vram_addr, + (u64)pagemap_addr[pos].addr, i - pos + incr); __fence = xe_migrate_from_vram(vr->migrate, i - pos + incr, vram_addr, - dma_addr + pos); + &pagemap_addr[pos]); } else { vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", - (u64)dma_addr[pos], vram_addr, i - pos + incr); + (u64)pagemap_addr[pos].addr, vram_addr, + i - pos + incr); __fence = xe_migrate_to_vram(vr->migrate, i - pos + incr, - dma_addr + pos, + &pagemap_addr[pos], vram_addr); } if (IS_ERR(__fence)) { @@ -423,7 +426,7 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, } /* Setup physical address of next device page */ - if (dma_addr[i] && spage) { + if (pagemap_addr[i].addr && spage) { vram_addr = __vram_addr; pos = i; } else { @@ -435,16 +438,16 @@ static int xe_svm_copy(struct page **pages, dma_addr_t *dma_addr, if (sram) { vm_dbg(&xe->drm, "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", - vram_addr, (u64)dma_addr[pos], 1); + vram_addr, (u64)pagemap_addr[pos].addr, 1); __fence = xe_migrate_from_vram(vr->migrate, 1, vram_addr, - dma_addr + pos); + &pagemap_addr[pos]); } else { vm_dbg(&xe->drm, "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", - (u64)dma_addr[pos], vram_addr, 1); + (u64)pagemap_addr[pos].addr, vram_addr, 1); __fence = xe_migrate_to_vram(vr->migrate, 1, - dma_addr + pos, + &pagemap_addr[pos], vram_addr); } if (IS_ERR(__fence)) { @@ -470,16 +473,18 @@ err_out: #undef XE_VRAM_ADDR_INVALID } -static int xe_svm_copy_to_devmem(struct page **pages, dma_addr_t *dma_addr, +static int xe_svm_copy_to_devmem(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages) { - return xe_svm_copy(pages, dma_addr, npages, XE_SVM_COPY_TO_VRAM); + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM); } -static int xe_svm_copy_to_ram(struct page **pages, dma_addr_t *dma_addr, +static int xe_svm_copy_to_ram(struct page **pages, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages) { - return xe_svm_copy(pages, dma_addr, npages, XE_SVM_COPY_TO_SRAM); + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM); } static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index 69d6ee49a3de..1d5919a99139 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -170,7 +170,7 @@ struct drm_pagemap_devmem_ops { /** * @copy_to_devmem: Copy to device memory (required for migration) * @pages: Pointer to array of device memory pages (destination) - * @dma_addr: Pointer to array of DMA addresses (source) + * @pagemap_addr: Pointer to array of DMA information (source) * @npages: Number of pages to copy * * Copy pages to device memory. @@ -178,13 +178,13 @@ struct drm_pagemap_devmem_ops { * Return: 0 on success, a negative error code on failure. */ int (*copy_to_devmem)(struct page **pages, - dma_addr_t *dma_addr, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages); /** * @copy_to_ram: Copy to system RAM (required for migration) * @pages: Pointer to array of device memory pages (source) - * @dma_addr: Pointer to array of DMA addresses (destination) + * @pagemap_addr: Pointer to array of DMA information (destination) * @npages: Number of pages to copy * * Copy pages to system RAM. @@ -192,7 +192,7 @@ struct drm_pagemap_devmem_ops { * Return: 0 on success, a negative error code on failure. */ int (*copy_to_ram)(struct page **pages, - dma_addr_t *dma_addr, + struct drm_pagemap_addr *pagemap_addr, unsigned long npages); }; From d755ff6063852cbd43c666726b69333d33d0d379 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:04 +0200 Subject: [PATCH 076/169] drm/pagemap: DMA map folios when possible If the page is part of a folio, DMA map the whole folio at once instead of mapping individual pages one after the other. For example if 2MB folios are used instead of 4KB pages, this reduces the number of DMA mappings by 512. The folio order (and consequently, the size) is persisted in the struct drm_pagemap_device_addr to be available at the time of unmapping. v2: - Initialize order variable (Matthew Brost) - Set proto and dir for completeness (Matthew Brost) - Do not populate drm_pagemap_addr, document it (Matthew Brost) - Add and use macro NR_PAGES(order) (Matthew Brost) Cc: Matthew Brost Reviewed-by: Matthew Brost Acked-by: Maarten Lankhorst Link: https://lore.kernel.org/r/20250805140028.599361-4-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/drm_pagemap.c | 25 ++++++++++++++++++------- include/drm/drm_pagemap.h | 10 ++++++++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index e5c135d793ed..0312edb8d4a8 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -222,24 +222,32 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, { unsigned long i; - for (i = 0; i < npages; ++i) { + for (i = 0; i < npages;) { struct page *page = migrate_pfn_to_page(migrate_pfn[i]); dma_addr_t dma_addr; + struct folio *folio; + unsigned int order = 0; if (!page) - continue; + goto next; if (WARN_ON_ONCE(is_zone_device_page(page))) return -EFAULT; - dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, dir); + folio = page_folio(page); + order = folio_order(folio); + + dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); if (dma_mapping_error(dev, dma_addr)) return -EFAULT; pagemap_addr[i] = drm_pagemap_addr_encode(dma_addr, DRM_INTERCONNECT_SYSTEM, - 0, dir); + order, dir); + +next: + i += NR_PAGES(order); } return 0; @@ -263,11 +271,14 @@ static void drm_pagemap_migrate_unmap_pages(struct device *dev, { unsigned long i; - for (i = 0; i < npages; ++i) { + for (i = 0; i < npages;) { if (!pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) - continue; + goto next; - dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE, dir); + dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE << pagemap_addr[i].order, dir); + +next: + i += NR_PAGES(pagemap_addr[i].order); } } diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index 1d5919a99139..f6e7e234c089 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -6,6 +6,8 @@ #include #include +#define NR_PAGES(order) (1U << (order)) + struct drm_pagemap; struct drm_pagemap_zdd; struct device; @@ -173,7 +175,9 @@ struct drm_pagemap_devmem_ops { * @pagemap_addr: Pointer to array of DMA information (source) * @npages: Number of pages to copy * - * Copy pages to device memory. + * Copy pages to device memory. If the order of a @pagemap_addr entry + * is greater than 0, the entry is populated but subsequent entries + * within the range of that order are not populated. * * Return: 0 on success, a negative error code on failure. */ @@ -187,7 +191,9 @@ struct drm_pagemap_devmem_ops { * @pagemap_addr: Pointer to array of DMA information (destination) * @npages: Number of pages to copy * - * Copy pages to system RAM. + * Copy pages to system RAM. If the order of a @pagemap_addr entry + * is greater than 0, the entry is populated but subsequent entries + * within the range of that order are not populated. * * Return: 0 on success, a negative error code on failure. */ From ddeda6136038b4b313bb6f9b44f2d363e1233814 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:05 +0200 Subject: [PATCH 077/169] drm/pagemap: Allocate folios when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the order is greater than zero, allocate a folio when populating the RAM PFNs instead of allocating individual pages one after the other. For example if 2MB folios are used instead of 4KB pages, this reduces the number of calls to the allocation API by 512. v2: - Use page order instead of extra argument (Matthew Brost) - Allocate with folio_alloc() (Matthew Brost) - Loop for mpages and free_pages based on order (Matthew Brost) v3: - Fix loops in drm_pagemap_migrate_populate_ram_pfn() (Matthew Brost) v4: - Use folio_trylock(), set local variable to NULL (Matthew Brost) Cc: Matthew Brost Reviewed-by: Matthew Brost Acked-by: Thomas Hellström Acked-by: Maarten Lankhorst Link: https://lore.kernel.org/r/20250805140028.599361-5-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/drm_pagemap.c | 58 +++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 0312edb8d4a8..22c44807e3fe 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -460,54 +460,80 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, { unsigned long i; - for (i = 0; i < npages; ++i, addr += PAGE_SIZE) { - struct page *page, *src_page; + for (i = 0; i < npages;) { + struct page *page = NULL, *src_page; + struct folio *folio; + unsigned int order = 0; if (!(src_mpfn[i] & MIGRATE_PFN_MIGRATE)) - continue; + goto next; src_page = migrate_pfn_to_page(src_mpfn[i]); if (!src_page) - continue; + goto next; if (fault_page) { if (src_page->zone_device_data != fault_page->zone_device_data) - continue; + goto next; } - if (vas) - page = alloc_page_vma(GFP_HIGHUSER, vas, addr); - else - page = alloc_page(GFP_HIGHUSER); + order = folio_order(page_folio(src_page)); - if (!page) + /* TODO: Support fallback to single pages if THP allocation fails */ + if (vas) + folio = vma_alloc_folio(GFP_HIGHUSER, order, vas, addr); + else + folio = folio_alloc(GFP_HIGHUSER, order); + + if (!folio) goto free_pages; + page = folio_page(folio, 0); mpfn[i] = migrate_pfn(page_to_pfn(page)); + +next: + if (page) + addr += page_size(page); + else + addr += PAGE_SIZE; + + i += NR_PAGES(order); } - for (i = 0; i < npages; ++i) { + for (i = 0; i < npages;) { struct page *page = migrate_pfn_to_page(mpfn[i]); + unsigned int order = 0; if (!page) - continue; + goto next_lock; - WARN_ON_ONCE(!trylock_page(page)); - ++*mpages; + WARN_ON_ONCE(!folio_trylock(page_folio(page))); + + order = folio_order(page_folio(page)); + *mpages += NR_PAGES(order); + +next_lock: + i += NR_PAGES(order); } return 0; free_pages: - for (i = 0; i < npages; ++i) { + for (i = 0; i < npages;) { struct page *page = migrate_pfn_to_page(mpfn[i]); + unsigned int order = 0; if (!page) - continue; + goto next_put; put_page(page); mpfn[i] = 0; + + order = folio_order(page_folio(page)); + +next_put: + i += NR_PAGES(order); } return -ENOMEM; } From 321d42032567e6ccbcec971b5547ae8ee1d4b861 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:06 +0200 Subject: [PATCH 078/169] drm/xe/migrate: Populate struct drm_pagemap_addr array Workaround to ensure all addresses are populated in the array as this is expected when creating the copy batch. This is required because the migrate layer does not support 2MB GPU pages yet. A proper fix will come in a follow-up. Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250805140028.599361-6-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_migrate.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 6a0f04411ae0..0f9636a06083 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -1802,6 +1802,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, unsigned int pitch = len >= PAGE_SIZE && !(len & ~PAGE_MASK) ? PAGE_SIZE : 4; int err; + unsigned long i, j; if (drm_WARN_ON(&xe->drm, (len & XE_CACHELINE_MASK) || (sram_offset | vram_addr) & XE_CACHELINE_MASK)) @@ -1818,6 +1819,24 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, return ERR_PTR(err); } + /* + * If the order of a struct drm_pagemap_addr entry is greater than 0, + * the entry is populated by GPU pagemap but subsequent entries within + * the range of that order are not populated. + * build_pt_update_batch_sram() expects a fully populated array of + * struct drm_pagemap_addr. Ensure this is the case even with higher + * orders. + */ + for (i = 0; i < npages;) { + unsigned int order = sram_addr[i].order; + + for (j = 1; j < NR_PAGES(order) && i + j < npages; j++) + if (!sram_addr[i + j].addr) + sram_addr[i + j].addr = sram_addr[i].addr + j * PAGE_SIZE; + + i += NR_PAGES(order); + } + build_pt_update_batch_sram(m, bb, pt_slot * XE_PAGE_SIZE, sram_addr, len + sram_offset); From 979f61633d069d761da606bef5efec42d33509e2 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 5 Aug 2025 15:59:07 +0200 Subject: [PATCH 079/169] drm/xe/svm: Migrate folios when possible The DMA mapping can now correspond to a folio (order > 0), so move the iterator by the number of pages in the folio in order to migrate all pages at once. This requires forcing contiguous memory for SVM BOs, which greatly simplifies the code and enables 2MB device page support, allowing a major performance improvement. Negative effects like extra eviction are unlikely as SVM BOs have a maximal size of 2MB. v2: - Improve commit message (Matthew Brost) - Fix increment, chunk, assert match (Matthew Brost) Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250805140028.599361-7-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_bo.c | 2 ++ drivers/gpu/drm/xe/xe_svm.c | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 1a72561f987c..6fea39842e1e 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -200,6 +200,8 @@ static bool force_contiguous(u32 bo_flags) else if (bo_flags & XE_BO_FLAG_PINNED && !(bo_flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) return true; /* needs vmap */ + else if (bo_flags & XE_BO_FLAG_CPU_ADDR_MIRROR) + return true; /* * For eviction / restore on suspend / resume objects pinned in VRAM diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 1d097e76aabc..e35c6d4def20 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -383,6 +383,14 @@ static int xe_svm_copy(struct page **pages, } match = vram_addr + PAGE_SIZE * (i - pos) == __vram_addr; + /* Expected with contiguous memory */ + xe_assert(vr->xe, match); + + if (pagemap_addr[i].order) { + i += NR_PAGES(pagemap_addr[i].order) - 1; + chunk = (i - pos) == (XE_MIGRATE_CHUNK_SIZE / PAGE_SIZE); + last = (i + 1) == npages; + } } /* From ad69d62588cd6bf8cddaff5e3e2eb1b8dd876d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Tue, 5 Aug 2025 11:18:50 +0200 Subject: [PATCH 080/169] drm/xe/pf: Program LMTT directory pointer on all GTs within a tile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the LMTT directory pointer was only programmed for primary GT within a tile. However, to ensure correct Local Memory access by VFs, the LMTT configuration must be programmed on all GTs within the tile. Lets program the LMTT directory pointer on every GT of the tile to guarantee proper LMEM access across all GTs on VFs. HSD: 18042797646 Bspec: 67468 Signed-off-by: Piotr Piórkowski Cc: Michal Wajdeczko Cc: Michał Winiarski Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250805091850.1508240-1-piotr.piorkowski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_lmtt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c index a2000307d5bf..a78c9d474a6e 100644 --- a/drivers/gpu/drm/xe/xe_lmtt.c +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -195,14 +195,17 @@ static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt) struct xe_tile *tile = lmtt_to_tile(lmtt); struct xe_device *xe = tile_to_xe(tile); dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE); + struct xe_gt *gt; + u8 id; lmtt_debug(lmtt, "DIR offset %pad\n", &offset); lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo)); lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K)); - xe_mmio_write32(&tile->mmio, - GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG, - LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K)); + for_each_gt_on_tile(gt, tile, id) + xe_mmio_write32(>->mmio, + GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG, + LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K)); } /** From 38b34e928a08ba594c4bbf7118aa3aadacd62fff Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Thu, 31 Jul 2025 10:38:09 +0100 Subject: [PATCH 081/169] drm/xe/migrate: prevent infinite recursion If the buf + offset is not aligned to XE_CAHELINE_BYTES we fallback to using a bounce buffer. However the bounce buffer here is allocated on the stack, and the only alignment requirement here is that it's naturally aligned to u8, and not XE_CACHELINE_BYTES. If the bounce buffer is also misaligned we then recurse back into the function again, however the new bounce buffer might also not be aligned, and might never be until we eventually blow through the stack, as we keep recursing. Instead of using the stack use kmalloc, which should respect the power-of-two alignment request here. Fixes a kernel panic when triggering this path through eudebug. v2 (Stuart): - Add build bug check for power-of-two restriction - s/EINVAL/ENOMEM/ Fixes: 270172f64b11 ("drm/xe: Update xe_ttm_access_memory to use GPU for non-visible access") Signed-off-by: Matthew Auld Cc: Maciej Patelczyk Cc: Stuart Summers Cc: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250731093807.207572-6-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 0f9636a06083..1e9ff18656db 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -2017,15 +2017,19 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, if (!IS_ALIGNED(len, XE_CACHELINE_BYTES) || !IS_ALIGNED((unsigned long)buf + offset, XE_CACHELINE_BYTES)) { int buf_offset = 0; + void *bounce; + int err; + + BUILD_BUG_ON(!is_power_of_2(XE_CACHELINE_BYTES)); + bounce = kmalloc(XE_CACHELINE_BYTES, GFP_KERNEL); + if (!bounce) + return -ENOMEM; /* * Less than ideal for large unaligned access but this should be * fairly rare, can fixup if this becomes common. */ do { - u8 bounce[XE_CACHELINE_BYTES]; - void *ptr = (void *)bounce; - int err; int copy_bytes = min_t(int, bytes_left, XE_CACHELINE_BYTES - (offset & XE_CACHELINE_MASK)); @@ -2034,22 +2038,22 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, err = xe_migrate_access_memory(m, bo, offset & ~XE_CACHELINE_MASK, - (void *)ptr, - sizeof(bounce), 0); + bounce, + XE_CACHELINE_BYTES, 0); if (err) - return err; + break; if (write) { - memcpy(ptr + ptr_offset, buf + buf_offset, copy_bytes); + memcpy(bounce + ptr_offset, buf + buf_offset, copy_bytes); err = xe_migrate_access_memory(m, bo, offset & ~XE_CACHELINE_MASK, - (void *)ptr, - sizeof(bounce), write); + bounce, + XE_CACHELINE_BYTES, write); if (err) - return err; + break; } else { - memcpy(buf + buf_offset, ptr + ptr_offset, + memcpy(buf + buf_offset, bounce + ptr_offset, copy_bytes); } @@ -2058,7 +2062,8 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, offset += copy_bytes; } while (bytes_left); - return 0; + kfree(bounce); + return err; } pagemap_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write); From 8c2d61e0e916e077fda7e7b8e67f25ffe0f361fc Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Thu, 31 Jul 2025 10:38:10 +0100 Subject: [PATCH 082/169] drm/xe/migrate: don't overflow max copy size With non-page aligned copy, we need to use 4 byte aligned pitch, however the size itself might still be close to our maximum of ~8M, and so the dimensions of the copy can easily exceed the S16_MAX limit of the copy command leading to the following assert: xe 0000:03:00.0: [drm] Assertion `size / pitch <= ((s16)(((u16)~0U) >> 1))` failed! platform: BATTLEMAGE subplatform: 1 graphics: Xe2_HPG 20.01 step A0 media: Xe2_HPM 13.01 step A1 tile: 0 VRAM 10.0 GiB GT: 0 type 1 WARNING: CPU: 23 PID: 10605 at drivers/gpu/drm/xe/xe_migrate.c:673 emit_copy+0x4b5/0x4e0 [xe] To fix this account for the pitch when calculating the number of current bytes to copy. Fixes: 270172f64b11 ("drm/xe: Update xe_ttm_access_memory to use GPU for non-visible access") Signed-off-by: Matthew Auld Cc: Maciej Patelczyk Cc: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250731093807.207572-7-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 1e9ff18656db..b36f5997706c 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -2084,6 +2084,12 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, else current_bytes = min_t(int, bytes_left, cursor.size); + if (current_bytes & ~PAGE_MASK) { + int pitch = 4; + + current_bytes = min_t(int, current_bytes, S16_MAX * pitch); + } + if (fence) dma_fence_put(fence); From 9b7ca35ed28fe5fad86e9d9c24ebd1271e4c9c3e Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Thu, 31 Jul 2025 10:38:11 +0100 Subject: [PATCH 083/169] drm/xe/migrate: prevent potential UAF If we hit the error path, the previous fence (if there is one) has already been put() prior to this, so doing a fence_wait could lead to UAF. Tweak the flow to do to the put() until after we do the wait. Fixes: 270172f64b11 ("drm/xe: Update xe_ttm_access_memory to use GPU for non-visible access") Signed-off-by: Matthew Auld Cc: Maciej Patelczyk Cc: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250731093807.207572-8-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index b36f5997706c..4effe10d815b 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -2090,9 +2090,6 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, current_bytes = min_t(int, current_bytes, S16_MAX * pitch); } - if (fence) - dma_fence_put(fence); - __fence = xe_migrate_vram(m, current_bytes, (unsigned long)buf & ~PAGE_MASK, &pagemap_addr[current_page], @@ -2100,11 +2097,15 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, XE_MIGRATE_COPY_TO_VRAM : XE_MIGRATE_COPY_TO_SRAM); if (IS_ERR(__fence)) { - if (fence) + if (fence) { dma_fence_wait(fence, false); + dma_fence_put(fence); + } fence = __fence; goto out_err; } + + dma_fence_put(fence); fence = __fence; buf += current_bytes; From 80944d334182ce5eb27d00e2bf20a88bfc32dea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Tue, 5 Aug 2025 09:48:42 +0200 Subject: [PATCH 084/169] drm/xe: Defer buffer object shrinker write-backs and GPU waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the xe buffer-object shrinker allows GPU waits and write-back, (typically from kswapd), perform multiple passes, skipping subsequent passes if the shrinker number of scanned objects target is reached. 1) Without GPU waits and write-back 2) Without write-back 3) With both GPU-waits and write-back This is to avoid stalls and costly write- and readbacks unless they are really necessary. v2: - Don't test for scan completion twice. (Stuart Summers) - Update tags. Reported-by: melvyn Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/5557 Cc: Summers Stuart Fixes: 00c8efc3180f ("drm/xe: Add a shrinker for xe bos") Cc: # v6.15+ Signed-off-by: Thomas Hellström Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250805074842.11359-1-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_shrinker.c | 51 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c index 125c836e0ee4..967448f683eb 100644 --- a/drivers/gpu/drm/xe/xe_shrinker.c +++ b/drivers/gpu/drm/xe/xe_shrinker.c @@ -54,10 +54,10 @@ xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgea write_unlock(&shrinker->lock); } -static s64 xe_shrinker_walk(struct xe_device *xe, - struct ttm_operation_ctx *ctx, - const struct xe_bo_shrink_flags flags, - unsigned long to_scan, unsigned long *scanned) +static s64 __xe_shrinker_walk(struct xe_device *xe, + struct ttm_operation_ctx *ctx, + const struct xe_bo_shrink_flags flags, + unsigned long to_scan, unsigned long *scanned) { unsigned int mem_type; s64 freed = 0, lret; @@ -87,6 +87,48 @@ static s64 xe_shrinker_walk(struct xe_device *xe, return freed; } +/* + * Try shrinking idle objects without writeback first, then if not sufficient, + * try also non-idle objects and finally if that's not sufficient either, + * add writeback. This avoids stalls and explicit writebacks with light or + * moderate memory pressure. + */ +static s64 xe_shrinker_walk(struct xe_device *xe, + struct ttm_operation_ctx *ctx, + const struct xe_bo_shrink_flags flags, + unsigned long to_scan, unsigned long *scanned) +{ + bool no_wait_gpu = true; + struct xe_bo_shrink_flags save_flags = flags; + s64 lret, freed; + + swap(no_wait_gpu, ctx->no_wait_gpu); + save_flags.writeback = false; + lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + swap(no_wait_gpu, ctx->no_wait_gpu); + if (lret < 0 || *scanned >= to_scan) + return lret; + + freed = lret; + if (!ctx->no_wait_gpu) { + lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + if (lret < 0) + return lret; + freed += lret; + if (*scanned >= to_scan) + return freed; + } + + if (flags.writeback) { + lret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned); + if (lret < 0) + return lret; + freed += lret; + } + + return freed; +} + static unsigned long xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) { @@ -193,6 +235,7 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup); shrink_flags.purge = false; + lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, nr_to_scan, &nr_scanned); if (lret >= 0) From 4d5c98eb77fe518944465d96e69de8aa6e9a5c99 Mon Sep 17 00:00:00 2001 From: Matt Atwood Date: Thu, 7 Aug 2025 14:42:23 -0700 Subject: [PATCH 085/169] drm/xe: rename XE_WA to XE_GT_WA Now that there are two types of wa tables and infrastructure, be more concise in the naming of GT wa macros. v2: update the documentation Signed-off-by: Matt Atwood Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250807214224.32728-1-matthew.s.atwood@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/display/intel_fbdev_fb.c | 2 +- drivers/gpu/drm/xe/display/xe_display_wa.c | 2 +- drivers/gpu/drm/xe/display/xe_plane_initial.c | 2 +- drivers/gpu/drm/xe/xe_device.c | 6 +++--- drivers/gpu/drm/xe/xe_eu_stall.c | 4 ++-- drivers/gpu/drm/xe/xe_ggtt.c | 8 ++++---- drivers/gpu/drm/xe/xe_gsc.c | 6 +++--- drivers/gpu/drm/xe/xe_gt.c | 6 +++--- drivers/gpu/drm/xe/xe_gt_topology.c | 2 +- drivers/gpu/drm/xe/xe_gt_types.h | 2 +- drivers/gpu/drm/xe/xe_guc.c | 14 ++++++------- drivers/gpu/drm/xe/xe_guc_ads.c | 20 +++++++++---------- drivers/gpu/drm/xe/xe_guc_pc.c | 10 +++++----- drivers/gpu/drm/xe/xe_hw_engine.c | 2 +- drivers/gpu/drm/xe/xe_lrc.c | 8 ++++---- drivers/gpu/drm/xe/xe_oa.c | 8 ++++---- drivers/gpu/drm/xe/xe_query.c | 4 ++-- drivers/gpu/drm/xe/xe_ring_ops.c | 6 +++--- drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 2 +- drivers/gpu/drm/xe/xe_wa.c | 2 +- drivers/gpu/drm/xe/xe_wa.h | 4 ++-- 22 files changed, 61 insertions(+), 61 deletions(-) diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c index b28a94df824f..70dbdf106fb6 100644 --- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c +++ b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c @@ -41,7 +41,7 @@ struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper, size = PAGE_ALIGN(size); obj = ERR_PTR(-ENODEV); - if (!IS_DGFX(xe) && !XE_WA(xe_root_mmio_gt(xe), 22019338487_display)) { + if (!IS_DGFX(xe) && !XE_GT_WA(xe_root_mmio_gt(xe), 22019338487_display)) { obj = xe_bo_create_pin_map(xe, xe_device_get_root_tile(xe), NULL, size, ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT | diff --git a/drivers/gpu/drm/xe/display/xe_display_wa.c b/drivers/gpu/drm/xe/display/xe_display_wa.c index 68d1387d81a0..8ada1cbcb16c 100644 --- a/drivers/gpu/drm/xe/display/xe_display_wa.c +++ b/drivers/gpu/drm/xe/display/xe_display_wa.c @@ -14,5 +14,5 @@ bool intel_display_needs_wa_16023588340(struct intel_display *display) { struct xe_device *xe = to_xe_device(display->drm); - return XE_WA(xe_root_mmio_gt(xe), 16023588340); + return XE_GT_WA(xe_root_mmio_gt(xe), 16023588340); } diff --git a/drivers/gpu/drm/xe/display/xe_plane_initial.c b/drivers/gpu/drm/xe/display/xe_plane_initial.c index 54ff70ea2d25..1fe06f4f9aca 100644 --- a/drivers/gpu/drm/xe/display/xe_plane_initial.c +++ b/drivers/gpu/drm/xe/display/xe_plane_initial.c @@ -122,7 +122,7 @@ initial_plane_bo(struct xe_device *xe, phys_base = base; flags |= XE_BO_FLAG_STOLEN; - if (XE_WA(xe_root_mmio_gt(xe), 22019338487_display)) + if (XE_GT_WA(xe_root_mmio_gt(xe), 22019338487_display)) return NULL; /* diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 57edbc63da6f..3e0402dff423 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -883,7 +883,7 @@ int xe_device_probe(struct xe_device *xe) } if (xe->tiles->media_gt && - XE_WA(xe->tiles->media_gt, 15015404425_disable)) + XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) XE_DEVICE_WA_DISABLE(xe, 15015404425); err = xe_devcoredump_init(xe); @@ -1043,7 +1043,7 @@ void xe_device_l2_flush(struct xe_device *xe) gt = xe_root_mmio_gt(xe); - if (!XE_WA(gt, 16023588340)) + if (!XE_GT_WA(gt, 16023588340)) return; fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); @@ -1087,7 +1087,7 @@ void xe_device_td_flush(struct xe_device *xe) return; root_gt = xe_root_mmio_gt(xe); - if (XE_WA(root_gt, 16023588340)) { + if (XE_GT_WA(root_gt, 16023588340)) { /* A transient flush is not sufficient: flush the L2 */ xe_device_l2_flush(xe); } else { diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index af7916315ac6..fdd514fec5ef 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -649,7 +649,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) return -ETIMEDOUT; } - if (XE_WA(gt, 22016596838)) + if (XE_GT_WA(gt, 22016596838)) xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, _MASKED_BIT_ENABLE(DISABLE_DOP_GATING)); @@ -805,7 +805,7 @@ static int xe_eu_stall_disable_locked(struct xe_eu_stall_data_stream *stream) cancel_delayed_work_sync(&stream->buf_poll_work); - if (XE_WA(gt, 22016596838)) + if (XE_GT_WA(gt, 22016596838)) xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2, _MASKED_BIT_DISABLE(DISABLE_DOP_GATING)); diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 249713139f69..e03222f5ac5a 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -106,10 +106,10 @@ static unsigned int probe_gsm_size(struct pci_dev *pdev) static void ggtt_update_access_counter(struct xe_ggtt *ggtt) { struct xe_tile *tile = ggtt->tile; - struct xe_gt *affected_gt = XE_WA(tile->primary_gt, 22019338487) ? + struct xe_gt *affected_gt = XE_GT_WA(tile->primary_gt, 22019338487) ? tile->primary_gt : tile->media_gt; struct xe_mmio *mmio = &affected_gt->mmio; - u32 max_gtt_writes = XE_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63; + u32 max_gtt_writes = XE_GT_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63; /* * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit * to wait for completion of prior GTT writes before letting this through. @@ -284,8 +284,8 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) if (GRAPHICS_VERx100(xe) >= 1270) ggtt->pt_ops = (ggtt->tile->media_gt && - XE_WA(ggtt->tile->media_gt, 22019338487)) || - XE_WA(ggtt->tile->primary_gt, 22019338487) ? + XE_GT_WA(ggtt->tile->media_gt, 22019338487)) || + XE_GT_WA(ggtt->tile->primary_gt, 22019338487) ? &xelpg_pt_wa_ops : &xelpg_pt_ops; else ggtt->pt_ops = &xelp_pt_ops; diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index 1d84bf2f2cef..f5ae28af60d4 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -266,7 +266,7 @@ static int gsc_upload_and_init(struct xe_gsc *gsc) unsigned int fw_ref; int ret; - if (XE_WA(tile->primary_gt, 14018094691)) { + if (XE_GT_WA(tile->primary_gt, 14018094691)) { fw_ref = xe_force_wake_get(gt_to_fw(tile->primary_gt), XE_FORCEWAKE_ALL); /* @@ -281,7 +281,7 @@ static int gsc_upload_and_init(struct xe_gsc *gsc) ret = gsc_upload(gsc); - if (XE_WA(tile->primary_gt, 14018094691)) + if (XE_GT_WA(tile->primary_gt, 14018094691)) xe_force_wake_put(gt_to_fw(tile->primary_gt), fw_ref); if (ret) @@ -593,7 +593,7 @@ void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep) u32 gs1_clr = prep ? 0 : HECI_H_GS1_ER_PREP; /* WA only applies if the GSC is loaded */ - if (!XE_WA(gt, 14015076503) || !gsc_fw_is_loaded(gt)) + if (!XE_GT_WA(gt, 14015076503) || !gsc_fw_is_loaded(gt)) return; xe_mmio_rmw32(>->mmio, HECI_H_GS1(MTL_GSC_HECI2_BASE), gs1_clr, gs1_set); diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 390394bbaadc..43f2da27d9d9 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -106,7 +106,7 @@ static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) unsigned int fw_ref; u32 reg; - if (!XE_WA(gt, 16023588340)) + if (!XE_GT_WA(gt, 16023588340)) return; fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); @@ -128,7 +128,7 @@ static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) unsigned int fw_ref; u32 reg; - if (!XE_WA(gt, 16023588340)) + if (!XE_GT_WA(gt, 16023588340)) return; if (xe_gt_is_media_type(gt)) @@ -966,7 +966,7 @@ int xe_gt_sanitize_freq(struct xe_gt *gt) if ((!xe_uc_fw_is_available(>->uc.gsc.fw) || xe_uc_fw_is_loaded(>->uc.gsc.fw) || xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) && - XE_WA(gt, 22019338487)) + XE_GT_WA(gt, 22019338487)) ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc); return ret; diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c index 8c63e3263643..a0baa560dd71 100644 --- a/drivers/gpu/drm/xe/xe_gt_topology.c +++ b/drivers/gpu/drm/xe/xe_gt_topology.c @@ -138,7 +138,7 @@ load_l3_bank_mask(struct xe_gt *gt, xe_l3_bank_mask_t l3_bank_mask) * but there's no tracking number assigned yet so we use a custom * OOB workaround descriptor. */ - if (XE_WA(gt, no_media_l3)) + if (XE_GT_WA(gt, no_media_l3)) return; if (GRAPHICS_VER(xe) >= 30) { diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index dfd4a16da5f0..ef0f2eecfa29 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -413,7 +413,7 @@ struct xe_gt { unsigned long *oob; /** * @wa_active.oob_initialized: mark oob as initialized to help - * detecting misuse of XE_WA() - it can only be called on + * detecting misuse of XE_GT_WA() - it can only be called on * initialization after OOB WAs have being processed */ bool oob_initialized; diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 9e34401e4489..433abc787f7b 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -157,7 +157,7 @@ static bool needs_wa_dual_queue(struct xe_gt *gt) * on RCS and CCSes with different address spaces, which on DG2 is * required as a WA for an HW bug. */ - if (XE_WA(gt, 22011391025)) + if (XE_GT_WA(gt, 22011391025)) return true; /* @@ -184,10 +184,10 @@ static u32 guc_ctl_wa_flags(struct xe_guc *guc) struct xe_gt *gt = guc_to_gt(guc); u32 flags = 0; - if (XE_WA(gt, 22012773006)) + if (XE_GT_WA(gt, 22012773006)) flags |= GUC_WA_POLLCS; - if (XE_WA(gt, 14014475959)) + if (XE_GT_WA(gt, 14014475959)) flags |= GUC_WA_HOLD_CCS_SWITCHOUT; if (needs_wa_dual_queue(gt)) @@ -201,17 +201,17 @@ static u32 guc_ctl_wa_flags(struct xe_guc *guc) if (GRAPHICS_VERx100(xe) < 1270) flags |= GUC_WA_PRE_PARSER; - if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685)) + if (XE_GT_WA(gt, 22012727170) || XE_GT_WA(gt, 22012727685)) flags |= GUC_WA_CONTEXT_ISOLATION; - if (XE_WA(gt, 18020744125) && + if (XE_GT_WA(gt, 18020744125) && !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER)) flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST; - if (XE_WA(gt, 1509372804)) + if (XE_GT_WA(gt, 1509372804)) flags |= GUC_WA_RENDER_RST_RC6_EXIT; - if (XE_WA(gt, 14018913170)) + if (XE_GT_WA(gt, 14018913170)) flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6; return flags; diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 2ceaa197cb2f..d7da67637079 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -247,7 +247,7 @@ static size_t calculate_regset_size(struct xe_gt *gt) count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES; - if (XE_WA(gt, 1607983814)) + if (XE_GT_WA(gt, 1607983814)) count += LNCFCMOCS_REG_COUNT; return count * sizeof(struct guc_mmio_reg); @@ -317,17 +317,17 @@ static void guc_waklv_init(struct xe_guc_ads *ads) offset = guc_ads_waklv_offset(ads); remain = guc_ads_waklv_size(ads); - if (XE_WA(gt, 14019882105) || XE_WA(gt, 16021333562)) + if (XE_GT_WA(gt, 14019882105) || XE_GT_WA(gt, 16021333562)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); - if (XE_WA(gt, 18024947630)) + if (XE_GT_WA(gt, 18024947630)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING); - if (XE_WA(gt, 16022287689)) + if (XE_GT_WA(gt, 16022287689)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE); - if (XE_WA(gt, 14022866841)) + if (XE_GT_WA(gt, 14022866841)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO); @@ -336,21 +336,21 @@ static void guc_waklv_init(struct xe_guc_ads *ads) * the default value for this register is determined to be 0xC40. This could change in the * future, so GuC depends on KMD to send it the correct value. */ - if (XE_WA(gt, 13011645652)) { + if (XE_GT_WA(gt, 13011645652)) { u32 data = 0xC40; guc_waklv_enable(ads, &data, sizeof(data) / sizeof(u32), &offset, &remain, GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE); } - if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406)) + if (XE_GT_WA(gt, 14022293748) || XE_GT_WA(gt, 22019794406)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); - if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_WA(gt, 16026508708)) + if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_GT_WA(gt, 16026508708)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); - if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_WA(gt, 16026007364)) { + if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_GT_WA(gt, 16026007364)) { u32 data[] = { 0x0, 0xF, @@ -761,7 +761,7 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, guc_mmio_regset_write_one(ads, regset_map, e->reg, count++); } - if (XE_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { + if (XE_GT_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { guc_mmio_regset_write_one(ads, regset_map, XELP_LNCFCMOCS(i), count++); diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 68b192fe3b32..0f8430acd16d 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -722,7 +722,7 @@ static int xe_guc_pc_set_max_freq_locked(struct xe_guc_pc *pc, u32 freq) */ int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq) { - if (XE_WA(pc_to_gt(pc), 22019338487)) { + if (XE_GT_WA(pc_to_gt(pc), 22019338487)) { if (wait_for_flush_complete(pc) != 0) return -EAGAIN; } @@ -835,7 +835,7 @@ static u32 pc_max_freq_cap(struct xe_guc_pc *pc) { struct xe_gt *gt = pc_to_gt(pc); - if (XE_WA(gt, 22019338487)) { + if (XE_GT_WA(gt, 22019338487)) { if (xe_gt_is_media_type(gt)) return min(LNL_MERT_FREQ_CAP, pc->rp0_freq); else @@ -899,7 +899,7 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) if (pc_get_min_freq(pc) > pc->rp0_freq) ret = pc_set_min_freq(pc, pc->rp0_freq); - if (XE_WA(tile->primary_gt, 14022085890)) + if (XE_GT_WA(tile->primary_gt, 14022085890)) ret = pc_set_min_freq(pc, max(BMG_MIN_FREQ, pc_get_min_freq(pc))); out: @@ -931,7 +931,7 @@ static bool needs_flush_freq_limit(struct xe_guc_pc *pc) { struct xe_gt *gt = pc_to_gt(pc); - return XE_WA(gt, 22019338487) && + return XE_GT_WA(gt, 22019338487) && pc->rp0_freq > BMG_MERT_FLUSH_FREQ_CAP; } @@ -1017,7 +1017,7 @@ static int pc_set_mert_freq_cap(struct xe_guc_pc *pc) { int ret; - if (!XE_WA(pc_to_gt(pc), 22019338487)) + if (!XE_GT_WA(pc_to_gt(pc), 22019338487)) return 0; guard(mutex)(&pc->freq_lock); diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 796ba8c34a16..1cf623b4a5bc 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -576,7 +576,7 @@ static void adjust_idledly(struct xe_hw_engine *hwe) u32 maxcnt_units_ns = 640; bool inhibit_switch = 0; - if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_WA(gt, 16023105232)) { + if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) { idledly = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base)); diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 8760c4c2ca38..8f6c3ba47882 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -75,7 +75,7 @@ lrc_to_xe(struct xe_lrc *lrc) static bool gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) { - if (XE_WA(gt, 16010904313) && + if (XE_GT_WA(gt, 16010904313) && (class == XE_ENGINE_CLASS_RENDER || class == XE_ENGINE_CLASS_COMPUTE)) return true; @@ -1071,7 +1071,7 @@ static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); u32 *cmd = batch; - if (!XE_WA(lrc->gt, 16010904313) || + if (!XE_GT_WA(lrc->gt, 16010904313) || !(hwe->class == XE_ENGINE_CLASS_RENDER || hwe->class == XE_ENGINE_CLASS_COMPUTE || hwe->class == XE_ENGINE_CLASS_COPY || @@ -1108,7 +1108,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, { u32 *cmd = batch; - if (!XE_WA(lrc->gt, 18022495364) || + if (!XE_GT_WA(lrc->gt, 18022495364) || hwe->class != XE_ENGINE_CLASS_RENDER) return 0; @@ -2093,7 +2093,7 @@ u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs) * continue to emit all of the SVG state since it's best not to leak * any of the state between contexts, even if that leakage is harmless. */ - if (XE_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { + if (XE_GT_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { state_table = xe_hpg_svg_state; state_table_size = ARRAY_SIZE(xe_hpg_svg_state); } diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 5729e7d3e335..a188bad172ad 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -822,7 +822,7 @@ static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) u32 sqcnt1; /* Enable thread stall DOP gating and EU DOP gating. */ - if (XE_WA(stream->gt, 1508761755)) { + if (XE_GT_WA(stream->gt, 1508761755)) { xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE)); xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, @@ -1079,7 +1079,7 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) * EU NOA signals behave incorrectly if EU clock gating is enabled. * Disable thread stall DOP gating and EU DOP gating. */ - if (XE_WA(stream->gt, 1508761755)) { + if (XE_GT_WA(stream->gt, 1508761755)) { xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, @@ -1754,7 +1754,7 @@ static int xe_oa_stream_init(struct xe_oa_stream *stream, * GuC reset of engines causes OA to lose configuration * state. Prevent this by overriding GUCRC mode. */ - if (XE_WA(stream->gt, 1509372804)) { + if (XE_GT_WA(stream->gt, 1509372804)) { ret = xe_guc_pc_override_gucrc_mode(>->uc.guc.pc, SLPC_GUCRC_MODE_GUCRC_NO_RC6); if (ret) @@ -1886,7 +1886,7 @@ u32 xe_oa_timestamp_frequency(struct xe_gt *gt) { u32 reg, shift; - if (XE_WA(gt, 18013179988) || XE_WA(gt, 14015568240)) { + if (XE_GT_WA(gt, 18013179988) || XE_GT_WA(gt, 14015568240)) { xe_pm_runtime_get(gt_to_xe(gt)); reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); xe_pm_runtime_put(gt_to_xe(gt)); diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 44d44bbc71dc..4dbe5732cb7f 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -477,7 +477,7 @@ static size_t calc_topo_query_size(struct xe_device *xe) sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss); /* L3bank mask may not be available for some GTs */ - if (!XE_WA(gt, no_media_l3)) + if (!XE_GT_WA(gt, no_media_l3)) query_size += sizeof(struct drm_xe_query_topology_mask) + sizeof_field(struct xe_gt, fuse_topo.l3_bank_mask); } @@ -540,7 +540,7 @@ static int query_gt_topology(struct xe_device *xe, * mask, then it's better to omit L3 from the query rather than * reporting bogus or zeroed information to userspace. */ - if (!XE_WA(gt, no_media_l3)) { + if (!XE_GT_WA(gt, no_media_l3)) { topo.type = DRM_XE_TOPO_L3_BANK; err = copy_mask(&query_ptr, &topo, gt->fuse_topo.l3_bank_mask, sizeof(gt->fuse_topo.l3_bank_mask)); diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index e8f22ec5f9af..5f15360d14bf 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -179,7 +179,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); u32 flags; - if (XE_WA(gt, 14016712196)) + if (XE_GT_WA(gt, 14016712196)) i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); @@ -190,7 +190,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) PIPE_CONTROL_DC_FLUSH_ENABLE | PIPE_CONTROL_FLUSH_ENABLE); - if (XE_WA(gt, 1409600907)) + if (XE_GT_WA(gt, 1409600907)) flags |= PIPE_CONTROL_DEPTH_STALL; if (lacks_render) @@ -206,7 +206,7 @@ static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int if (hwe->class != XE_ENGINE_CLASS_RENDER) return i; - if (XE_WA(hwe->gt, 16020292621)) + if (XE_GT_WA(hwe->gt, 16020292621)) i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, RING_NOPID(hwe->mmio_base).addr, 0); diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c index 9a9733447230..dc588255674d 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -166,7 +166,7 @@ static u32 detect_bar2_integrated(struct xe_device *xe, struct xe_ttm_stolen_mgr stolen_size -= wopcm_size; - if (media_gt && XE_WA(media_gt, 14019821291)) { + if (media_gt && XE_GT_WA(media_gt, 14019821291)) { u64 gscpsmi_base = xe_mmio_read64_2x32(&media_gt->mmio, GSCPSMI_BASE) & ~GENMASK_ULL(5, 0); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 432ea325677d..148a2425006f 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2068,7 +2068,7 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->extensions)) return -EINVAL; - if (XE_WA(xe_root_mmio_gt(xe), 14016763929)) + if (XE_GT_WA(xe_root_mmio_gt(xe), 14016763929)) args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 535067e7fb0c..0fc73af1bb4c 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -1090,6 +1090,6 @@ void xe_wa_apply_tile_workarounds(struct xe_tile *tile) if (IS_SRIOV_VF(tile->xe)) return; - if (XE_WA(tile->primary_gt, 22010954014)) + if (XE_GT_WA(tile->primary_gt, 22010954014)) xe_mmio_rmw32(mmio, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS); } diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h index f3880c65cb8d..105497c1d7d7 100644 --- a/drivers/gpu/drm/xe/xe_wa.h +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -25,11 +25,11 @@ void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p); void xe_wa_dump(struct xe_gt *gt, struct drm_printer *p); /** - * XE_WA - Out-of-band workarounds, to be queried and called as needed. + * XE_GT_WA - Out-of-band GT workarounds, to be queried and called as needed. * @gt__: gt instance * @id__: XE_OOB_, as generated by build system in generated/xe_wa_oob.h */ -#define XE_WA(gt__, id__) ({ \ +#define XE_GT_WA(gt__, id__) ({ \ xe_gt_assert(gt__, (gt__)->wa_active.oob_initialized); \ test_bit(XE_WA_OOB_ ## id__, (gt__)->wa_active.oob); \ }) From 342d1f84320994ab72a6ca96e18f2985868e1cbd Mon Sep 17 00:00:00 2001 From: Matt Atwood Date: Thu, 7 Aug 2025 14:42:24 -0700 Subject: [PATCH 086/169] drm/xe: Update function names for GT specific workarounds Now that there distinctly different OOB functions, update the names to reflect the IPs they interact with. Signed-off-by: Matt Atwood Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250807214224.32728-2-matthew.s.atwood@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_gt.c | 4 ++-- drivers/gpu/drm/xe/xe_wa.c | 10 +++++----- drivers/gpu/drm/xe/xe_wa.h | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 43f2da27d9d9..a3397f04abcc 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -400,7 +400,7 @@ int xe_gt_init_early(struct xe_gt *gt) xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); - err = xe_wa_init(gt); + err = xe_wa_gt_init(gt); if (err) return err; @@ -408,7 +408,7 @@ int xe_gt_init_early(struct xe_gt *gt) if (err) return err; - xe_wa_process_oob(gt); + xe_wa_process_gt_oob(gt); xe_force_wake_init_gt(gt, gt_to_fw(gt)); spin_lock_init(>->global_invl_lock); diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 0fc73af1bb4c..52c7df4c3afd 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -916,13 +916,13 @@ void xe_wa_process_device_oob(struct xe_device *xe) } /** - * xe_wa_process_oob - process OOB workaround table + * xe_wa_process_gt_oob - process GT OOB workaround table * @gt: GT instance to process workarounds for * * Process OOB workaround table for this platform, marking in @gt the * workarounds that are active. */ -void xe_wa_process_oob(struct xe_gt *gt) +void xe_wa_process_gt_oob(struct xe_gt *gt) { struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); @@ -1006,12 +1006,12 @@ int xe_wa_device_init(struct xe_device *xe) } /** - * xe_wa_init - initialize gt with workaround bookkeeping + * xe_wa_gt_init - initialize gt with workaround bookkeeping * @gt: GT instance to initialize * * Returns 0 for success, negative error code otherwise. */ -int xe_wa_init(struct xe_gt *gt) +int xe_wa_gt_init(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); size_t n_oob, n_lrc, n_engine, n_gt, total; @@ -1037,7 +1037,7 @@ int xe_wa_init(struct xe_gt *gt) return 0; } -ALLOW_ERROR_INJECTION(xe_wa_init, ERRNO); /* See xe_pci_probe() */ +ALLOW_ERROR_INJECTION(xe_wa_gt_init, ERRNO); /* See xe_pci_probe() */ void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p) { diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h index 105497c1d7d7..6a869b2de643 100644 --- a/drivers/gpu/drm/xe/xe_wa.h +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -14,9 +14,9 @@ struct xe_hw_engine; struct xe_tile; int xe_wa_device_init(struct xe_device *xe); -int xe_wa_init(struct xe_gt *gt); +int xe_wa_gt_init(struct xe_gt *gt); void xe_wa_process_device_oob(struct xe_device *xe); -void xe_wa_process_oob(struct xe_gt *gt); +void xe_wa_process_gt_oob(struct xe_gt *gt); void xe_wa_process_gt(struct xe_gt *gt); void xe_wa_process_engine(struct xe_hw_engine *hwe); void xe_wa_process_lrc(struct xe_hw_engine *hwe); From 9f8aa0bcd1f35baefe083adfa26bc6f9f68f652b Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Fri, 8 Aug 2025 13:06:28 +0530 Subject: [PATCH 087/169] drm/xe/vf: Refactor CCS save/restore to use default migration context Previously, CCS save/restore operations created separate migration contexts with new VM memory allocations, resulting in significant overhead. This commit eliminates redundant context creation reusing the default migration context by registering new execution queues for CCS save and restore on the existing migrate VM. Signed-off-by: Satyanarayana K V P Suggested-by: Matthew Brost Cc: Michal Wajdeczko Cc: John Harrison Reviewed-by: Matthew Brost Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250808073628.32745-2-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 15 +++++++ drivers/gpu/drm/xe/xe_exec_queue.h | 1 + drivers/gpu/drm/xe/xe_migrate.c | 21 +++++---- drivers/gpu/drm/xe/xe_migrate.h | 2 +- drivers/gpu/drm/xe/xe_pm.c | 3 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 51 ++++++++++++---------- drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h | 6 +-- 7 files changed, 63 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 6c176183ed58..2d10a53f701d 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -789,6 +789,21 @@ int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, return ret; } +/** + * xe_exec_queue_lrc() - Get the LRC from exec queue. + * @q: The exec_queue. + * + * Retrieves the primary LRC for the exec queue. Note that this function + * returns only the first LRC instance, even when multiple parallel LRCs + * are configured. + * + * Return: Pointer to LRC on success, error on failure + */ +struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) +{ + return q->lrc[0]; +} + /** * xe_exec_queue_is_lr() - Whether an exec_queue is long-running * @q: The exec_queue diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 4d416f23001c..15ec852e7f7e 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -94,4 +94,5 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q); +struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q); #endif diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 4effe10d815b..ac9f4d4988d2 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -951,7 +951,7 @@ err_sync: } /** - * xe_get_migrate_lrc() - Get the LRC from migrate context. + * xe_migrate_lrc() - Get the LRC from migrate context. * @migrate: Migrate context. * * Return: Pointer to LRC on success, error on failure @@ -961,14 +961,15 @@ struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate) return migrate->q->lrc[0]; } -static int emit_flush_invalidate(struct xe_migrate *m, u32 *dw, int i, +static int emit_flush_invalidate(struct xe_exec_queue *q, u32 *dw, int i, u32 flags) { + struct xe_lrc *lrc = xe_exec_queue_lrc(q); dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | flags; - dw[i++] = lower_32_bits(xe_lrc_start_seqno_ggtt_addr(xe_migrate_lrc(m))) | + dw[i++] = lower_32_bits(xe_lrc_start_seqno_ggtt_addr(lrc)) | MI_FLUSH_DW_USE_GTT; - dw[i++] = upper_32_bits(xe_lrc_start_seqno_ggtt_addr(xe_migrate_lrc(m))); + dw[i++] = upper_32_bits(xe_lrc_start_seqno_ggtt_addr(lrc)); dw[i++] = MI_NOOP; dw[i++] = MI_NOOP; @@ -977,7 +978,8 @@ static int emit_flush_invalidate(struct xe_migrate *m, u32 *dw, int i, /** * xe_migrate_ccs_rw_copy() - Copy content of TTM resources. - * @m: The migration context. + * @tile: Tile whose migration context to be used. + * @q : Execution to be used along with migration context. * @src_bo: The buffer object @src is currently bound to. * @read_write : Creates BB commands for CCS read/write. * @@ -988,7 +990,7 @@ static int emit_flush_invalidate(struct xe_migrate *m, u32 *dw, int i, * * Return: 0 if successful, negative error code on failure. */ -int xe_migrate_ccs_rw_copy(struct xe_migrate *m, +int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, enum xe_sriov_vf_ccs_rw_ctxs read_write) @@ -996,7 +998,8 @@ int xe_migrate_ccs_rw_copy(struct xe_migrate *m, bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; struct ttm_resource *src = src_bo->ttm.resource; - struct xe_gt *gt = m->tile->primary_gt; + struct xe_migrate *m = tile->migrate; + struct xe_gt *gt = tile->primary_gt; u32 batch_size, batch_size_allocated; struct xe_device *xe = gt_to_xe(gt); struct xe_res_cursor src_it, ccs_it; @@ -1079,11 +1082,11 @@ int xe_migrate_ccs_rw_copy(struct xe_migrate *m, emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); - bb->len = emit_flush_invalidate(m, bb->cs, bb->len, flush_flags); + bb->len = emit_flush_invalidate(q, bb->cs, bb->len, flush_flags); flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt, src_L0_ofs, dst_is_pltt, src_L0, ccs_ofs, true); - bb->len = emit_flush_invalidate(m, bb->cs, bb->len, flush_flags); + bb->len = emit_flush_invalidate(q, bb->cs, bb->len, flush_flags); size -= src_L0; } diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index a3f3fa281e04..8978d2cc1a75 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -125,7 +125,7 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, struct ttm_resource *dst, bool copy_only_ccs); -int xe_migrate_ccs_rw_copy(struct xe_migrate *m, +int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, enum xe_sriov_vf_ccs_rw_ctxs read_write); diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index ed29791beb7c..cc489c41ae30 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -209,6 +209,9 @@ int xe_pm_resume(struct xe_device *xe) xe_pxp_pm_resume(xe->pxp); + if (IS_SRIOV_VF(xe)) + xe_sriov_vf_ccs_register_context(xe); + drm_dbg(&xe->drm, "Device resumed\n"); return 0; err: diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index f0ca2a9b2bb7..4872e43eb440 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -8,6 +8,7 @@ #include "xe_bb.h" #include "xe_bo.h" #include "xe_device.h" +#include "xe_exec_queue.h" #include "xe_exec_queue_types.h" #include "xe_guc_submit.h" #include "xe_lrc.h" @@ -168,8 +169,8 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_tile_vf_ccs *ctx) static void ccs_rw_update_ring(struct xe_tile_vf_ccs *ctx) { - struct xe_lrc *lrc = xe_migrate_lrc(ctx->migrate); u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); u32 dw[10], i = 0; dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; @@ -183,13 +184,12 @@ static void ccs_rw_update_ring(struct xe_tile_vf_ccs *ctx) xe_lrc_set_ring_tail(lrc, lrc->ring.tail); } -static int register_save_restore_context(struct xe_migrate *m, - enum xe_sriov_vf_ccs_rw_ctxs ctx_id) +static int register_save_restore_context(struct xe_tile_vf_ccs *ctx) { int err = -EINVAL; int ctx_type; - switch (ctx_id) { + switch (ctx->ctx_id) { case XE_SRIOV_VF_CCS_READ_CTX: ctx_type = GUC_CONTEXT_COMPRESSION_SAVE; break; @@ -200,7 +200,7 @@ static int register_save_restore_context(struct xe_migrate *m, return err; } - xe_guc_register_exec_queue(xe_migrate_exec_queue(m), ctx_type); + xe_guc_register_exec_queue(ctx->mig_q, ctx_type); return 0; } @@ -225,7 +225,7 @@ int xe_sriov_vf_ccs_register_context(struct xe_device *xe) for_each_ccs_rw_ctx(ctx_id) { ctx = &tile->sriov.vf.ccs[ctx_id]; - err = register_save_restore_context(ctx->migrate, ctx_id); + err = register_save_restore_context(ctx); if (err) return err; } @@ -236,13 +236,14 @@ int xe_sriov_vf_ccs_register_context(struct xe_device *xe) static void xe_sriov_vf_ccs_fini(void *arg) { struct xe_tile_vf_ccs *ctx = arg; - struct xe_lrc *lrc = xe_migrate_lrc(ctx->migrate); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); /* * Make TAIL = HEAD in the ring so that no issues are seen if Guc * submits this context to HW on VF pause after unbinding device. */ xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc)); + xe_exec_queue_put(ctx->mig_q); } /** @@ -258,8 +259,9 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) { struct xe_tile *tile = xe_device_get_root_tile(xe); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; - struct xe_migrate *migrate; struct xe_tile_vf_ccs *ctx; + struct xe_exec_queue *q; + u32 flags; int err; xe_assert(xe, IS_SRIOV_VF(xe)); @@ -270,37 +272,40 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) ctx = &tile->sriov.vf.ccs[ctx_id]; ctx->ctx_id = ctx_id; - migrate = xe_migrate_alloc(tile); - if (!migrate) { - err = -ENOMEM; + flags = EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_PERMANENT | + EXEC_QUEUE_FLAG_MIGRATE; + q = xe_exec_queue_create_bind(xe, tile, flags, 0); + if (IS_ERR(q)) { + err = PTR_ERR(q); goto err_ret; } - - err = xe_migrate_init(migrate); - if (err) - goto err_ret; - - ctx->migrate = migrate; + ctx->mig_q = q; err = alloc_bb_pool(tile, ctx); if (err) - goto err_ret; + goto err_free_queue; ccs_rw_update_ring(ctx); - err = register_save_restore_context(ctx->migrate, ctx_id); + err = register_save_restore_context(ctx); if (err) - goto err_ret; + goto err_free_queue; err = devm_add_action_or_reset(xe->drm.dev, xe_sriov_vf_ccs_fini, ctx); + if (err) + goto err_ret; } xe->sriov.vf.ccs.initialized = 1; return 0; +err_free_queue: + xe_exec_queue_put(q); + err_ret: return err; } @@ -319,7 +324,7 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) { struct xe_device *xe = xe_bo_device(bo); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; - struct xe_migrate *migrate; + struct xe_tile_vf_ccs *ctx; struct xe_tile *tile; struct xe_bb *bb; int err = 0; @@ -334,8 +339,8 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) /* bb should be NULL here. Assert if not NULL */ xe_assert(xe, !bb); - migrate = tile->sriov.vf.ccs[ctx_id].migrate; - err = xe_migrate_ccs_rw_copy(migrate, bo, ctx_id); + ctx = &tile->sriov.vf.ccs[ctx_id]; + err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id); } return err; } diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h index e240f3fd18af..93435a6f4cb6 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h @@ -41,11 +41,11 @@ struct xe_sa_manager; struct xe_tile_vf_ccs { /** @id: Id to which context it belongs to */ enum xe_sriov_vf_ccs_rw_ctxs ctx_id; - /** @migrate: Migration helper for save/restore of CCS data */ - struct xe_migrate *migrate; + /** @mig_q: exec queues used for migration */ + struct xe_exec_queue *mig_q; struct { - /** @ccs_rw_bb_pool: Pool from which batch buffers are allocated. */ + /** @ccs_bb_pool: Pool from which batch buffers are allocated. */ struct xe_sa_manager *ccs_bb_pool; } mem; }; From d301eb950da59f962bafe874cf5eb6d61a85b2c2 Mon Sep 17 00:00:00 2001 From: Karthik Poosa Date: Sat, 9 Aug 2025 00:23:10 +0530 Subject: [PATCH 088/169] drm/xe/hwmon: Add SW clamp for power limits writes Clamp writes to power limits powerX_crit/currX_crit, powerX_cap, powerX_max, to the maximum supported by the pcode mailbox when sysfs-provided values exceed this limit. Although the pcode already performs clamping, values beyond the pcode mailbox's supported range get truncated, leading to incorrect critical power settings. This patch ensures proper clamping to prevent such truncation. v2: - Address below review comments. (Riana) - Split comments into multiple sentences. - Use local variables for readability. - Add a debug log. - Use u64 instead of unsigned long. v3: - Change drm_dbg logs to drm_info. (Badal) v4: - Rephrase the drm_info log. (Rodrigo, Riana) - Rename variable max_mbx_power_limit to max_supp_power_limit, as limit is same for platforms with and without mailbox power limit support. Signed-off-by: Karthik Poosa Fixes: 92d44a422d0d ("drm/xe/hwmon: Expose card reactive critical power") Fixes: fb1b70607f73 ("drm/xe/hwmon: Expose power attributes") Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250808185310.3466529-1-karthik.poosa@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_hwmon.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index f08fc4377d25..c17ed1ae8649 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -332,6 +332,7 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channe int ret = 0; u32 reg_val, max; struct xe_reg rapl_limit; + u64 max_supp_power_limit = 0; mutex_lock(&hwmon->hwmon_lock); @@ -356,6 +357,20 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channe goto unlock; } + /* + * If the sysfs value exceeds the maximum pcode supported power limit value, clamp it to + * the supported maximum (U12.3 format). + * This is to avoid truncation during reg_val calculation below and ensure the valid + * power limit is sent for pcode which would clamp it to card-supported value. + */ + max_supp_power_limit = ((PWR_LIM_VAL) >> hwmon->scl_shift_power) * SF_POWER; + if (value > max_supp_power_limit) { + value = max_supp_power_limit; + drm_info(&hwmon->xe->drm, + "Power limit clamped as selected %s exceeds channel %d limit\n", + PWR_ATTR_TO_STR(attr), channel); + } + /* Computation in 64-bits to avoid overflow. Round to nearest. */ reg_val = DIV_ROUND_CLOSEST_ULL((u64)value << hwmon->scl_shift_power, SF_POWER); @@ -739,9 +754,23 @@ static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, int channel, { int ret; u32 uval; + u64 max_crit_power_curr = 0; mutex_lock(&hwmon->hwmon_lock); + /* + * If the sysfs value exceeds the pcode mailbox cmd POWER_SETUP_SUBCOMMAND_WRITE_I1 + * max supported value, clamp it to the command's max (U10.6 format). + * This is to avoid truncation during uval calculation below and ensure the valid power + * limit is sent for pcode which would clamp it to card-supported value. + */ + max_crit_power_curr = (POWER_SETUP_I1_DATA_MASK >> POWER_SETUP_I1_SHIFT) * scale_factor; + if (value > max_crit_power_curr) { + value = max_crit_power_curr; + drm_info(&hwmon->xe->drm, + "Power limit clamped as selected exceeds channel %d limit\n", + channel); + } uval = DIV_ROUND_CLOSEST_ULL(value << POWER_SETUP_I1_SHIFT, scale_factor); ret = xe_hwmon_pcode_write_i1(hwmon, uval); From edb660ad79ffe81a982c2eca02360a6ffac83e46 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Fri, 8 Aug 2025 10:41:08 -0700 Subject: [PATCH 089/169] drm/intel/pciids: Add match on vendor/id only All our PCI ID macros match on the PCI class besides the vendor and devid, even for devices that may or may not have display. This may not work going forward, so add a simple INTEL_PCI_DEVICE that matches only on vendor/device IDs. Cc: Jani Nikula Reviewed-by: Jani Nikula Link: https://lore.kernel.org/r/20250808-intel-pci-device-v1-1-ce3545d86502@intel.com Signed-off-by: Lucas De Marchi --- include/drm/intel/pciids.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h index 76f8d26f9cc9..da6301a6fcea 100644 --- a/include/drm/intel/pciids.h +++ b/include/drm/intel/pciids.h @@ -26,6 +26,11 @@ #define __PCIIDS_H__ #ifdef __KERNEL__ +#define INTEL_PCI_DEVICE(_id, _info) { \ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, (_id)), \ + .driver_data = (kernel_ulong_t)(_info), \ +} + #define INTEL_VGA_DEVICE(_id, _info) { \ PCI_DEVICE(PCI_VENDOR_ID_INTEL, (_id)), \ .class = PCI_BASE_CLASS_DISPLAY << 16, .class_mask = 0xff << 16, \ From 17593a69b75f098280ad88b625f2d8c5bfe4c6a1 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 8 Aug 2025 11:34:56 +0100 Subject: [PATCH 090/169] drm/xe: rework PDE PAT index selection For non-leaf paging structures we end up selecting a random index between [0, 3], depending on the first user if the page-table is shared, since non-leaf structures only have two bits in the HW for encoding the PAT index, and here we are just passing along the full user provided index, which can be an index as large as ~31 on xe2+. The user provided index is meant for the leaf node, which maps the actual BO pages where we have more PAT bits, and not the non-leaf nodes which are only mapping other paging structures, and so only needs a minimal PAT index range. Also the chosen index might need to consider how the driver mapped the paging structures on the host side, like wc vs wb, which is separate from the user provided index. With that move the PDE PAT index selection under driver control. For now just use a coherent index on platforms with page-tables that are cached on host side, and incoherent otherwise. Using a coherent index could potentially be expensive, and would be overkill if we know the page-table is always uncached on host side. v2 (Stuart): - Add some documentation and split into separate helper. BSpec: 59510 Signed-off-by: Matthew Auld Cc: Stuart Summers Cc: Matthew Brost Reviewed-by: Stuart Summers Link: https://lore.kernel.org/r/20250808103455.462424-2-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 10 ++++------ drivers/gpu/drm/xe/xe_pt.c | 4 ++-- drivers/gpu/drm/xe/xe_pt_types.h | 3 +-- drivers/gpu/drm/xe/xe_vm.c | 34 +++++++++++++++++++++++++++----- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index ac9f4d4988d2..45cde272d45f 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -152,8 +152,7 @@ static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, for (pos = dpa_base; pos < vram_limit; pos += SZ_1G, ofs += 8) { if (pos + SZ_1G >= vram_limit) { - entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs, - pat_index); + entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs); xe_map_wr(xe, &bo->vmap, ofs, u64, entry); flags = vm->pt_ops->pte_encode_addr(xe, 0, @@ -207,7 +206,7 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, /* PT30 & PT31 reserved for 2M identity map */ pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE; - entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index); + entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs); xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; @@ -275,15 +274,14 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, flags = XE_PDE_64K; entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * - XE_PAGE_SIZE, pat_index); + XE_PAGE_SIZE); xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, entry | flags); } /* Write PDE's that point to our BO. */ for (i = 0; i < map_ofs / PAGE_SIZE; i++) { - entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, - pat_index); + entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE); xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + (i + 1) * 8, u64, entry); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 330cc0f54a3f..f3a39e734a90 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -69,7 +69,7 @@ static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, if (level > MAX_HUGEPTE_LEVEL) return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo, - 0, pat_index); + 0); return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) | XE_PTE_NULL; @@ -616,7 +616,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, xe_child->is_compact = true; } - pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags; + pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags; ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, pte); } diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h index 69eab6f37cfe..17cdd7c7e9f5 100644 --- a/drivers/gpu/drm/xe/xe_pt_types.h +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -45,8 +45,7 @@ struct xe_pt_ops { u64 (*pte_encode_addr)(struct xe_device *xe, u64 addr, u16 pat_index, u32 pt_level, bool devmem, u64 flags); - u64 (*pde_encode_bo)(struct xe_bo *bo, u64 bo_offset, - u16 pat_index); + u64 (*pde_encode_bo)(struct xe_bo *bo, u64 bo_offset); }; struct xe_pt_entry { diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 148a2425006f..97913f496a46 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1512,14 +1512,39 @@ static u64 pte_encode_ps(u32 pt_level) return 0; } -static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset, - const u16 pat_index) +static u16 pde_pat_index(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + u16 pat_index; + + /* + * We only have two bits to encode the PAT index in non-leaf nodes, but + * these only point to other paging structures so we only need a minimal + * selection of options. The user PAT index is only for encoding leaf + * nodes, where we have use of more bits to do the encoding. The + * non-leaf nodes are instead under driver control so the chosen index + * here should be distict from the user PAT index. Also the + * corresponding coherency of the PAT index should be tied to the + * allocation type of the page table (or at least we should pick + * something which is always safe). + */ + if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) + pat_index = xe->pat.idx[XE_CACHE_WB]; + else + pat_index = xe->pat.idx[XE_CACHE_NONE]; + + xe_assert(xe, pat_index <= 3); + + return pat_index; +} + +static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset) { u64 pde; pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); pde |= XE_PAGE_PRESENT | XE_PAGE_RW; - pde |= pde_encode_pat_index(pat_index); + pde |= pde_encode_pat_index(pde_pat_index(bo)); return pde; } @@ -2024,8 +2049,7 @@ struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) { - return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0, - tile_to_xe(tile)->pat.idx[XE_CACHE_WB]); + return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); } static struct xe_exec_queue * From 9337166fa1d80f7bb7c7d3a8f901f21c348c0f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 11 Aug 2025 12:43:57 +0200 Subject: [PATCH 091/169] drm/xe: Assign ioctl xe file handler to vm in xe_vm_create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In several code paths, such as xe_pt_create(), the vm->xef field is used to determine whether a VM originates from userspace or the kernel. Previously, this handler was only assigned in xe_vm_create_ioctl(), after the VM was created by xe_vm_create(). However, xe_vm_create() triggers page table creation, and that function assumes vm->xef should be already set. This could lead to incorrect origin detection. To fix this problem and ensure consistency in the initialization of the VM object, let's move the assignment of this handler to xe_vm_create. v2: - take reference to the xe file object only when xef is not NULL - release the reference to the xe file object on the error path (Matthew) Fixes: 7f387e6012b6 ("drm/xe: add XE_BO_FLAG_PINNED_LATE_RESTORE") Signed-off-by: Piotr Piórkowski Cc: Matthew Auld Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20250811104358.2064150-2-piotr.piorkowski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_migrate.c | 2 +- drivers/gpu/drm/xe/xe_pxp_submit.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 11 ++++++----- drivers/gpu/drm/xe/xe_vm.h | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 45cde272d45f..ddfad7506a82 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -409,7 +409,7 @@ int xe_migrate_init(struct xe_migrate *m) /* Special layout, prepared below.. */ vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | - XE_VM_FLAG_SET_TILE_ID(tile)); + XE_VM_FLAG_SET_TILE_ID(tile), NULL); if (IS_ERR(vm)) return PTR_ERR(vm); diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.c b/drivers/gpu/drm/xe/xe_pxp_submit.c index d92ec0f515b0..ca95f2a4d4ef 100644 --- a/drivers/gpu/drm/xe/xe_pxp_submit.c +++ b/drivers/gpu/drm/xe/xe_pxp_submit.c @@ -101,7 +101,7 @@ static int allocate_gsc_client_resources(struct xe_gt *gt, xe_assert(xe, hwe); /* PXP instructions must be issued from PPGTT */ - vm = xe_vm_create(xe, XE_VM_FLAG_GSC); + vm = xe_vm_create(xe, XE_VM_FLAG_GSC, NULL); if (IS_ERR(vm)) return PTR_ERR(vm); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 97913f496a46..6c52a0a610c8 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1665,7 +1665,7 @@ static void xe_vm_free_scratch(struct xe_vm *vm) } } -struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) { struct drm_gem_object *vm_resv_obj; struct xe_vm *vm; @@ -1686,9 +1686,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) vm->xe = xe; vm->size = 1ull << xe->info.va_bits; - vm->flags = flags; + if (xef) + vm->xef = xe_file_get(xef); /** * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be * manipulated under the PXP mutex. However, the PXP mutex can be taken @@ -1839,6 +1840,8 @@ err_no_resv: for_each_tile(tile, xe, id) xe_range_fence_tree_fini(&vm->rftree[id]); ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); + if (vm->xef) + xe_file_put(vm->xef); kfree(vm); if (flags & XE_VM_FLAG_LR_MODE) xe_pm_runtime_put(xe); @@ -2121,7 +2124,7 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) flags |= XE_VM_FLAG_FAULT_MODE; - vm = xe_vm_create(xe, flags); + vm = xe_vm_create(xe, flags, xef); if (IS_ERR(vm)) return PTR_ERR(vm); @@ -2137,8 +2140,6 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, vm->usm.asid = asid; } - vm->xef = xe_file_get(xef); - /* Record BO memory for VM pagetable created against client */ for_each_tile(tile, xe, id) if (vm->pt_root[id]) diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 3475a118f666..2f213737c7e5 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -26,7 +26,7 @@ struct xe_sync_entry; struct xe_svm_range; struct drm_exec; -struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags); +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef); struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id); int xe_vma_cmp_vma_cb(const void *key, const struct rb_node *node); From 30e0c3f43a414616e0b6ca76cf7f7b2cd387e1d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Mon, 11 Aug 2025 12:43:58 +0200 Subject: [PATCH 092/169] drm/xe: Move ASID allocation and user PT BO tracking into xe_vm_create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, ASID assignment for user VMs and page-table BO accounting for client memory tracking are performed in xe_vm_create_ioctl. To consolidate VM object initialization, move this logic to xe_vm_create. v2: - removed unnecessary duplicate BO tracking code - using the local variable xef to verify whether the VM is being created by userspace Suggested-by: Matthew Auld Signed-off-by: Piotr Piórkowski Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20250811104358.2064150-3-piotr.piorkowski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_vm.c | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 6c52a0a610c8..d40d2d43c041 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1820,6 +1820,20 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) if (number_tiles > 1) vm->composite_fence_ctx = dma_fence_context_alloc(1); + if (xef && xe->info.has_asid) { + u32 asid; + + down_write(&xe->usm.lock); + err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, + XA_LIMIT(1, XE_MAX_ASID - 1), + &xe->usm.next_asid, GFP_KERNEL); + up_write(&xe->usm.lock); + if (err < 0) + goto err_unlock_close; + + vm->usm.asid = asid; + } + trace_xe_vm_create(vm); return vm; @@ -2086,9 +2100,8 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); struct drm_xe_vm_create *args = data; - struct xe_tile *tile; struct xe_vm *vm; - u32 id, asid; + u32 id; int err; u32 flags = 0; @@ -2128,23 +2141,6 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (IS_ERR(vm)) return PTR_ERR(vm); - if (xe->info.has_asid) { - down_write(&xe->usm.lock); - err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, - XA_LIMIT(1, XE_MAX_ASID - 1), - &xe->usm.next_asid, GFP_KERNEL); - up_write(&xe->usm.lock); - if (err < 0) - goto err_close_and_put; - - vm->usm.asid = asid; - } - - /* Record BO memory for VM pagetable created against client */ - for_each_tile(tile, xe, id) - if (vm->pt_root[id]) - xe_drm_client_add_bo(vm->xef->client, vm->pt_root[id]->bo); - #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) /* Warning: Security issue - never enable by default */ args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); From 32a4d1b98e6663101fd0abfaf151c48feea7abb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Tue, 27 May 2025 14:06:37 +0200 Subject: [PATCH 093/169] drm/xe/pf: Set VF LMEM BAR size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LMEM is partitioned between multiple VFs and we expect that the more VFs we have, the less LMEM is assigned to each VF. This means that we can achieve full LMEM BAR access without the need to attempt full VF LMEM BAR resize via pci_resize_resource(). Always try to set the largest possible BAR size that allows to fit the number of enabled VFs and inform the user in case the resize attempt is not successful. Signed-off-by: Michał Winiarski Reviewed-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20250527120637.665506-7-michal.winiarski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/regs/xe_bars.h | 1 + drivers/gpu/drm/xe/xe_pci_sriov.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_bars.h b/drivers/gpu/drm/xe/regs/xe_bars.h index ce05b6ae832f..880140d6ccdc 100644 --- a/drivers/gpu/drm/xe/regs/xe_bars.h +++ b/drivers/gpu/drm/xe/regs/xe_bars.h @@ -7,5 +7,6 @@ #define GTTMMADR_BAR 0 /* MMIO + GTT */ #define LMEM_BAR 2 /* VRAM */ +#define VF_LMEM_BAR 9 /* VF VRAM */ #endif diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 447a7867eecb..af05db07162e 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -3,6 +3,10 @@ * Copyright © 2023-2024 Intel Corporation */ +#include +#include + +#include "regs/xe_bars.h" #include "xe_assert.h" #include "xe_device.h" #include "xe_gt_sriov_pf_config.h" @@ -128,6 +132,18 @@ static void pf_engine_activity_stats(struct xe_device *xe, unsigned int num_vfs, } } +static int resize_vf_vram_bar(struct xe_device *xe, int num_vfs) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + u32 sizes; + + sizes = pci_iov_vf_bar_get_sizes(pdev, VF_LMEM_BAR, num_vfs); + if (!sizes) + return 0; + + return pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, __fls(sizes)); +} + static int pf_enable_vfs(struct xe_device *xe, int num_vfs) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); @@ -158,6 +174,12 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) if (err < 0) goto failed; + if (IS_DGFX(xe)) { + err = resize_vf_vram_bar(xe, num_vfs); + if (err) + xe_sriov_info(xe, "Failed to set VF LMEM BAR size: %d\n", err); + } + err = pci_enable_sriov(pdev, num_vfs); if (err < 0) goto failed; From cc8b221172eff821c54d1012bc0c3414638b86cf Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 14 Aug 2025 14:48:45 +0530 Subject: [PATCH 094/169] drm/xe/i2c: Introduce xe_i2c_present() In preparation of wider usecases which require checking for I2C controller presence, introduce xe_i2c_present() helper. v2: Doc fix (Riana) Suggested-by: Riana Tauro Signed-off-by: Raag Jadav Reviewed-by: Heikki Krogerus Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250814091845.1356135-1-raag.jadav@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_i2c.c | 18 ++++++++++++++++-- drivers/gpu/drm/xe/xe_i2c.h | 2 ++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index bc7dc2099470..044dda517b7c 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -146,6 +146,20 @@ static void xe_i2c_unregister_adapter(struct xe_i2c *i2c) fwnode_remove_software_node(i2c->adapter_node); } +/** + * xe_i2c_present - I2C controller is present and functional + * @xe: xe device instance + * + * Check whether the I2C controller is present and functioning with valid + * endpoint cookie. + * + * Return: %true if present, %false otherwise. + */ +bool xe_i2c_present(struct xe_device *xe) +{ + return xe->i2c && xe->i2c->ep.cookie == XE_I2C_EP_COOKIE_DEVICE; +} + /** * xe_i2c_irq_handler: Handler for I2C interrupts * @xe: xe device instance @@ -230,7 +244,7 @@ void xe_i2c_pm_suspend(struct xe_device *xe) { struct xe_mmio *mmio = xe_root_tile_mmio(xe); - if (!xe->i2c || xe->i2c->ep.cookie != XE_I2C_EP_COOKIE_DEVICE) + if (!xe_i2c_present(xe)) return; xe_mmio_rmw32(mmio, I2C_CONFIG_PMCSR, PCI_PM_CTRL_STATE_MASK, (__force u32)PCI_D3hot); @@ -241,7 +255,7 @@ void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) { struct xe_mmio *mmio = xe_root_tile_mmio(xe); - if (!xe->i2c || xe->i2c->ep.cookie != XE_I2C_EP_COOKIE_DEVICE) + if (!xe_i2c_present(xe)) return; if (d3cold) diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h index b767ed8ce52b..ecd5f10358e2 100644 --- a/drivers/gpu/drm/xe/xe_i2c.h +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -49,11 +49,13 @@ struct xe_i2c { #if IS_ENABLED(CONFIG_I2C) int xe_i2c_probe(struct xe_device *xe); +bool xe_i2c_present(struct xe_device *xe); void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl); void xe_i2c_pm_suspend(struct xe_device *xe); void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold); #else static inline int xe_i2c_probe(struct xe_device *xe) { return 0; } +static inline bool xe_i2c_present(struct xe_device *xe) { return false; } static inline void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl) { } static inline void xe_i2c_pm_suspend(struct xe_device *xe) { } static inline void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) { } From 14fcd7361ed1c74dad4062313f2863b643409f6f Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 12 Aug 2025 22:06:13 +0530 Subject: [PATCH 095/169] drm/xe/pm: Disable RPM for SR-IOV VFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VFs without native PCIe Power Management (PM) capabilities inherit their PF's power state as per PCIe specifications(§5.10.1 PCIe Base Spec 7.0). Enabling Runtime Power Management (RPM) for these VFs trigger unnecessary driver suspend/resume operations that ultimately perform no PCI-level power transition. Since VFs without PM capabilities cannot independently enter low-power states, the existing RPM workflow becomes redundant: 1. Driver executes full suspend/resume sequence 2. PCI PM transition step becomes no-op 3. VF power state remains tied to PF's status Disabling RPM for VFs eliminates this redundant processing while maintaining proper power management through PF dependency. This optimization ensures VFs follow their PF's power state without superfluous runtime handling. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Michał Winiarski Cc: Anshuman Gupta Cc: Rodrigo Vivi Cc: Maarten Lankhorst Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250812163613.9954-1-satyanarayana.k.v.p@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pm.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index cc489c41ae30..51ed7270994a 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -247,6 +247,10 @@ static void xe_pm_runtime_init(struct xe_device *xe) { struct device *dev = xe->drm.dev; + /* Our current VFs do not support RPM. so, disable it */ + if (IS_SRIOV_VF(xe)) + return; + /* * Disable the system suspend direct complete optimization. * We need to ensure that the regular device suspend/resume functions @@ -371,6 +375,10 @@ static void xe_pm_runtime_fini(struct xe_device *xe) { struct device *dev = xe->drm.dev; + /* Our current VFs do not support RPM. so, disable it */ + if (IS_SRIOV_VF(xe)) + return; + pm_runtime_get_sync(dev); pm_runtime_forbid(dev); } From a01b704527c28a2fd43a17a85f8996b75ec8492a Mon Sep 17 00:00:00 2001 From: Christoph Manszewski Date: Wed, 13 Aug 2025 12:12:30 +0200 Subject: [PATCH 096/169] drm/xe: Fix vm_bind_ioctl double free bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the argument check during an array bind fails, the bind_ops are freed twice as seen below. Fix this by setting bind_ops to NULL after freeing. ================================================================== BUG: KASAN: double-free in xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] Free of addr ffff88813bb9b800 by task xe_vm/14198 CPU: 5 UID: 0 PID: 14198 Comm: xe_vm Not tainted 6.16.0-xe-eudebug-cmanszew+ #520 PREEMPT(full) Hardware name: Intel Corporation Alder Lake Client Platform/AlderLake-P DDR5 RVP, BIOS ADLPFWI1.R00.2411.A02.2110081023 10/08/2021 Call Trace: dump_stack_lvl+0x82/0xd0 print_report+0xcb/0x610 ? __virt_addr_valid+0x19a/0x300 ? xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] kasan_report_invalid_free+0xc8/0xf0 ? xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] ? xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] check_slab_allocation+0x102/0x130 kfree+0x10d/0x440 ? should_fail_ex+0x57/0x2f0 ? xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] xe_vm_bind_ioctl+0x1b2/0x21f0 [xe] ? __pfx_xe_vm_bind_ioctl+0x10/0x10 [xe] ? __lock_acquire+0xab9/0x27f0 ? lock_acquire+0x165/0x300 ? drm_dev_enter+0x53/0xe0 [drm] ? find_held_lock+0x2b/0x80 ? drm_dev_exit+0x30/0x50 [drm] ? drm_ioctl_kernel+0x128/0x1c0 [drm] drm_ioctl_kernel+0x128/0x1c0 [drm] ? __pfx_xe_vm_bind_ioctl+0x10/0x10 [xe] ? find_held_lock+0x2b/0x80 ? __pfx_drm_ioctl_kernel+0x10/0x10 [drm] ? should_fail_ex+0x57/0x2f0 ? __pfx_xe_vm_bind_ioctl+0x10/0x10 [xe] drm_ioctl+0x352/0x620 [drm] ? __pfx_drm_ioctl+0x10/0x10 [drm] ? __pfx_rpm_resume+0x10/0x10 ? do_raw_spin_lock+0x11a/0x1b0 ? find_held_lock+0x2b/0x80 ? __pm_runtime_resume+0x61/0xc0 ? rcu_is_watching+0x20/0x50 ? trace_irq_enable.constprop.0+0xac/0xe0 xe_drm_ioctl+0x91/0xc0 [xe] __x64_sys_ioctl+0xb2/0x100 ? rcu_is_watching+0x20/0x50 do_syscall_64+0x68/0x2e0 entry_SYSCALL_64_after_hwframe+0x76/0x7e RIP: 0033:0x7fa9acb24ded Fixes: b43e864af0d4 ("drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR") Cc: Matthew Brost Cc: Himal Prasad Ghimiray Cc: Thomas Hellström Signed-off-by: Christoph Manszewski Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250813101231.196632-2-christoph.manszewski@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index d40d2d43c041..6fd28b205453 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3442,6 +3442,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, free_bind_ops: if (args->num_binds > 1) kvfree(*bind_ops); + *bind_ops = NULL; return err; } @@ -3548,7 +3549,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) struct xe_exec_queue *q = NULL; u32 num_syncs, num_ufence = 0; struct xe_sync_entry *syncs = NULL; - struct drm_xe_vm_bind_op *bind_ops; + struct drm_xe_vm_bind_op *bind_ops = NULL; struct xe_vma_ops vops; struct dma_fence *fence; int err; From aa81d55a5997b190660e21e72177f1542c4e3d59 Mon Sep 17 00:00:00 2001 From: Christoph Manszewski Date: Wed, 13 Aug 2025 12:12:31 +0200 Subject: [PATCH 097/169] drm/xe: Untangle vm_bind_ioctl cleanup order Change the resource cleanup order to align with reverse allocation order. Cc: Matthew Brost Signed-off-by: Christoph Manszewski Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250813101231.196632-3-christoph.manszewski@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 6fd28b205453..c86337e08a55 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3567,7 +3567,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) q = xe_exec_queue_lookup(xef, args->exec_queue_id); if (XE_IOCTL_DBG(xe, !q)) { err = -ENOENT; - goto put_vm; + goto free_bind_ops; } if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { @@ -3613,7 +3613,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) __GFP_RETRY_MAYFAIL | __GFP_NOWARN); if (!ops) { err = -ENOMEM; - goto release_vm_lock; + goto free_bos; } } @@ -3747,17 +3747,20 @@ free_syncs: put_obj: for (i = 0; i < args->num_binds; ++i) xe_bo_put(bos[i]); + + kvfree(ops); +free_bos: + kvfree(bos); release_vm_lock: up_write(&vm->lock); put_exec_queue: if (q) xe_exec_queue_put(q); -put_vm: - xe_vm_put(vm); - kvfree(bos); - kvfree(ops); +free_bind_ops: if (args->num_binds > 1) kvfree(bind_ops); +put_vm: + xe_vm_put(vm); return err; } From b85bb2d677153d990924d31be9416166d22382eb Mon Sep 17 00:00:00 2001 From: Simon Richter Date: Mon, 18 Aug 2025 15:47:35 +0900 Subject: [PATCH 098/169] drm/xe: Make page size consistent in loop If PAGE_SIZE != XE_PAGE_SIZE (which is currently locked behind CONFIG_BROKEN), this would generate the wrong number of PDEs. Since these PDEs are consumed by the GPU, the GPU page size needs to be used. Signed-off-by: Simon Richter Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250818064806.2835-1-Simon.Richter@hogyros.de --- drivers/gpu/drm/xe/xe_migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index ddfad7506a82..57e6d5a8ac39 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -280,7 +280,7 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, } /* Write PDE's that point to our BO. */ - for (i = 0; i < map_ofs / PAGE_SIZE; i++) { + for (i = 0; i < map_ofs / XE_PAGE_SIZE; i++) { entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE); xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + From 6627be03d3e1195b22172554191365378c3460e7 Mon Sep 17 00:00:00 2001 From: Nitin Gote Date: Tue, 19 Aug 2025 11:41:51 +0530 Subject: [PATCH 099/169] drm/xe/tuning: Apply "Disable NULL query for Anyhit Shader" to Xe2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the "Disable NULL query for Anyhit Shader" tuning to Xe2 (graphics version 2000+) platforms, in addition to Xe3. This sets the DIS_NULL_QUERY bit in RT_CTRL to disable null query for Anyhit shaders on both Xe2 and Xe3. This is a change in behavior that can regress a userspace not prepared for it. However it's not feasible to change dynamically the option per client or per exec queue via an opt-in flag. Mesa is already prepared for that and it got propagated to their stable versions. Even if it was possible, at this point adding a flag would mean mesa would also need to propagate such a flag to their stable versions, otherwise the previous fix would not be used. Signed-off-by: Nitin Gote Acked-by: Tapani Pälli Link: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35044 Reviewed-by: Lucas De Marchi Link: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35044 Link: https://lore.kernel.org/r/20250819061151.1272622-1-nitin.r.gote@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_tuning.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c index 828b45b24c23..a524170a04d0 100644 --- a/drivers/gpu/drm/xe/xe_tuning.c +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -99,7 +99,7 @@ static const struct xe_rtp_entry_sr engine_tunings[] = { XE_RTP_ACTIONS(SET(SAMPLER_MODE, INDIRECT_STATE_BASE_ADDR_OVERRIDE)) }, { XE_RTP_NAME("Tuning: Disable NULL query for Anyhit Shader"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, XE_RTP_END_VERSION_UNDEFINED), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED), FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(RT_CTRL, DIS_NULL_QUERY)) }, From 97a18f8349423371650996fc506dfd6cc4d3e1ad Mon Sep 17 00:00:00 2001 From: Gustavo Sousa Date: Mon, 18 Aug 2025 15:15:46 -0300 Subject: [PATCH 100/169] drm/xe: Probe for tile count during device info initialization The function mmio_multi_tile_setup() does not look like the proper location for probing for the number of existing tiles in the hardware. It should not be that function's responsibility and such information should instead be already available when it gets called. The fact that we need to adjust gt_count is a symptom of that. Move probing of available tile count to a dedicated function named xe_info_probe_tile_count() and call it from xe_info_init(), which seems like a more appropriate place for that. With that move, we no longer need to (and shouldn't) adjust gt_count as a part of xe_info_probe_tile_count(), as that field will be initialized later in xe_info_init(). v4: - Only probe for tile count if the default tile_count != 1 (just like was done in mmio_multi_tile_setup()). (CI) v3: - Unchanged. v2: - Use KUnit static stub so that we do not try to query hardware when running KUnit tests. (CI) - Tweak last paragraph of commit message to make it clearer. (Jonathan) Reviewed-by: Jonathan Cavitt Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250818-gt_count-improvements-v4-1-ee12870c6f57@intel.com Signed-off-by: Gustavo Sousa --- drivers/gpu/drm/xe/tests/xe_pci.c | 7 ++++++ drivers/gpu/drm/xe/xe_mmio.c | 33 ------------------------- drivers/gpu/drm/xe/xe_pci.c | 41 +++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index 9c715e59f030..db30c5156d0c 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -101,6 +101,11 @@ static void fake_read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, } } +static void fake_xe_info_probe_tile_count(struct xe_device *xe) +{ + /* Nothing to do, just use the statically defined value. */ +} + int xe_pci_fake_device_init(struct xe_device *xe) { struct kunit *test = kunit_get_current_test(); @@ -138,6 +143,8 @@ done: data->sriov_mode : XE_SRIOV_MODE_NONE; kunit_activate_static_stub(test, read_gmdid, fake_read_gmdid); + kunit_activate_static_stub(test, xe_info_probe_tile_count, + fake_xe_info_probe_tile_count); xe_info_init_early(xe, desc, subplatform_desc); xe_info_init(xe, desc); diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index e4db8d58ea2d..ef6f3ea573a2 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -58,7 +58,6 @@ static void tiles_fini(void *arg) static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) { struct xe_tile *tile; - struct xe_gt *gt; u8 id; /* @@ -68,38 +67,6 @@ static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) if (xe->info.tile_count == 1) return; - /* Possibly override number of tile based on configuration register */ - if (!xe->info.skip_mtcfg) { - struct xe_mmio *mmio = xe_root_tile_mmio(xe); - u8 tile_count, gt_count; - u32 mtcfg; - - /* - * Although the per-tile mmio regs are not yet initialized, this - * is fine as it's going to the root tile's mmio, that's - * guaranteed to be initialized earlier in xe_mmio_probe_early() - */ - mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR); - tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; - - if (tile_count < xe->info.tile_count) { - drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", - xe->info.tile_count, tile_count); - xe->info.tile_count = tile_count; - - /* - * We've already setup gt_count according to the full - * tile count. Re-calculate it to only include the GTs - * that belong to the remaining tile(s). - */ - gt_count = 0; - for_each_gt(gt, xe, id) - if (gt->info.id < tile_count * xe->info.max_gt_per_tile) - gt_count++; - xe->info.gt_count = gt_count; - } - } - for_each_remote_tile(tile, xe, id) xe_mmio_init(&tile->mmio, tile, xe->mmio.regs + id * tile_mmio_size, SZ_4M); } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 053faa7f9b01..bc73fe176165 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -17,6 +17,7 @@ #include "display/xe_display.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_regs.h" #include "xe_device.h" #include "xe_drv.h" #include "xe_gt.h" @@ -603,6 +604,44 @@ static int xe_info_init_early(struct xe_device *xe, return 0; } +/* + * Possibly override number of tile based on configuration register. + */ +static void xe_info_probe_tile_count(struct xe_device *xe) +{ + struct xe_mmio *mmio; + u8 tile_count; + u32 mtcfg; + + KUNIT_STATIC_STUB_REDIRECT(xe_info_probe_tile_count, xe); + + /* + * Probe for tile count only for platforms that support multiple + * tiles. + */ + if (xe->info.tile_count == 1) + return; + + if (xe->info.skip_mtcfg) + return; + + mmio = xe_root_tile_mmio(xe); + + /* + * Although the per-tile mmio regs are not yet initialized, this + * is fine as it's going to the root tile's mmio, that's + * guaranteed to be initialized earlier in xe_mmio_probe_early() + */ + mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR); + tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; + + if (tile_count < xe->info.tile_count) { + drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", + xe->info.tile_count, tile_count); + xe->info.tile_count = tile_count; + } +} + /* * Initialize device info content that does require knowledge about * graphics / media IP version. @@ -677,6 +716,8 @@ static int xe_info_init(struct xe_device *xe, xe->info.has_usm = graphics_desc->has_usm; xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; + xe_info_probe_tile_count(xe); + for_each_remote_tile(tile, xe, id) { int err; From 1046be1db620d00eb390a94ac7b67109e0beeff5 Mon Sep 17 00:00:00 2001 From: Gustavo Sousa Date: Mon, 18 Aug 2025 15:15:47 -0300 Subject: [PATCH 101/169] drm/xe: Use for_each_gt to define gt_count We are currently bumping gt_count as we define GTs for each tile. While that works with our current code, there are reasons to improve that: * That section of the code is dedicated to define each tile's set of GTs and their respective info. The gt_count can be seen as a device level property. While it is fair to bump it as we define each GT, we can also focus on the GT themselves and count after we are done with the definitions. * More *importantly*, gt_count should reflect the number of GTs that we would get when looping over them with for_each_gt(). Bumping gt_count the way we are currently doing makes that value not really connected to for_each_gt(). This could bite us in the future if in the loop gets extra checks that are not accounted for in each existing "gt_count++". As such, let's use for_each_gt() and extract the calculation of gt_count into a separate block, just after we define the set of GTs for each tile. Reviewed-by: Jonathan Cavitt Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250818-gt_count-improvements-v4-2-ee12870c6f57@intel.com Signed-off-by: Gustavo Sousa --- drivers/gpu/drm/xe/xe_pci.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index bc73fe176165..419e29cebbba 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -740,7 +740,6 @@ static int xe_info_init(struct xe_device *xe, gt->info.id = tile->id * xe->info.max_gt_per_tile; gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state; gt->info.engine_mask = graphics_desc->hw_engine_mask; - xe->info.gt_count++; err = xe_tile_alloc_vram(tile); if (err) @@ -765,9 +764,15 @@ static int xe_info_init(struct xe_device *xe, gt->info.id = tile->id * xe->info.max_gt_per_tile + 1; gt->info.has_indirect_ring_state = media_desc->has_indirect_ring_state; gt->info.engine_mask = media_desc->hw_engine_mask; - xe->info.gt_count++; } + /* + * Now that we have tiles and GTs defined, let's loop over valid GTs + * in order to define gt_count. + */ + for_each_gt(gt, xe, id) + xe->info.gt_count++; + return 0; } From bc2b2062682a7f8d828b0020e37a84a9234f03a6 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 18 Aug 2025 21:20:32 +0200 Subject: [PATCH 102/169] drm/xe/kunit: Extend platform generator with PTL Our list of typical platforms used to generate test device objects does not contain any PANTHERLAKE. Add one. Signed-off-by: Michal Wajdeczko Cc: Lucas De Marchi Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250818192032.633-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_wa_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c index c96d1fe34151..416258c193f6 100644 --- a/drivers/gpu/drm/xe/tests/xe_wa_test.c +++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c @@ -75,6 +75,7 @@ static const struct platform_test_case cases[] = { GMDID_CASE(LUNARLAKE, 2004, A0, 2000, A0), GMDID_CASE(LUNARLAKE, 2004, B0, 2000, A0), GMDID_CASE(BATTLEMAGE, 2001, A0, 1301, A1), + GMDID_CASE(PANTHERLAKE, 3000, A0, 3000, A0), }; static void platform_desc(const struct platform_test_case *t, char *desc) From 2515d2b9ab4108c11a0b23935e68de27abb8b2a7 Mon Sep 17 00:00:00 2001 From: Stuart Summers Date: Tue, 19 Aug 2025 20:10:54 +0000 Subject: [PATCH 103/169] drm/xe/pcode: Initialize data0 for pcode read routine There are two registers filled in when reading data from pcode besides the mailbox itself. Currently, we allow a NULL value for the second of these two (data1) and assume the first is defined. However, many of the routines that are calling this function assume that pcode will ignore the value being passed in and so leave that first value (data0) defined but uninitialized. To be safe, make sure this value is always initialized to something (0 generally) in the event pcode behavior changes and starts using this value. v2: Fix sob/author Signed-off-by: Stuart Summers Reviewed-by: Jonathan Cavitt Link: https://lore.kernel.org/r/20250819201054.393220-1-stuart.summers@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device_sysfs.c | 8 ++++---- drivers/gpu/drm/xe/xe_hwmon.c | 8 ++++---- drivers/gpu/drm/xe/xe_vram_freq.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c index bd9015761aa0..6ee422594b56 100644 --- a/drivers/gpu/drm/xe/xe_device_sysfs.c +++ b/drivers/gpu/drm/xe/xe_device_sysfs.c @@ -76,7 +76,7 @@ lb_fan_control_version_show(struct device *dev, struct device_attribute *attr, c { struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); struct xe_tile *root = xe_device_get_root_tile(xe); - u32 cap, ver_low = FAN_TABLE, ver_high = FAN_TABLE; + u32 cap = 0, ver_low = FAN_TABLE, ver_high = FAN_TABLE; u16 major = 0, minor = 0, hotfix = 0, build = 0; int ret; @@ -115,7 +115,7 @@ lb_voltage_regulator_version_show(struct device *dev, struct device_attribute *a { struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); struct xe_tile *root = xe_device_get_root_tile(xe); - u32 cap, ver_low = VR_CONFIG, ver_high = VR_CONFIG; + u32 cap = 0, ver_low = VR_CONFIG, ver_high = VR_CONFIG; u16 major = 0, minor = 0, hotfix = 0, build = 0; int ret; @@ -153,7 +153,7 @@ static int late_bind_create_files(struct device *dev) { struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); struct xe_tile *root = xe_device_get_root_tile(xe); - u32 cap; + u32 cap = 0; int ret; xe_pm_runtime_get(xe); @@ -186,7 +186,7 @@ static void late_bind_remove_files(struct device *dev) { struct xe_device *xe = pdev_to_xe_device(to_pci_dev(dev)); struct xe_tile *root = xe_device_get_root_tile(xe); - u32 cap; + u32 cap = 0; int ret; xe_pm_runtime_get(xe); diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index c17ed1ae8649..32a76ae6e9dc 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -179,7 +179,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr u32 clr, u32 set) { struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); - u32 val0, val1; + u32 val0 = 0, val1 = 0; int ret = 0; ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_POWER_SETUP, @@ -734,7 +734,7 @@ static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel, long *value, u32 scale_factor) { int ret; - u32 uval; + u32 uval = 0; mutex_lock(&hwmon->hwmon_lock); @@ -918,7 +918,7 @@ xe_hwmon_power_write(struct xe_hwmon *hwmon, u32 attr, int channel, long val) static umode_t xe_hwmon_curr_is_visible(const struct xe_hwmon *hwmon, u32 attr, int channel) { - u32 uval; + u32 uval = 0; /* hwmon sysfs attribute of current available only for package */ if (channel != CHANNEL_PKG) @@ -1020,7 +1020,7 @@ xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) static umode_t xe_hwmon_fan_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) { - u32 uval; + u32 uval = 0; if (!hwmon->xe->info.has_fan_control) return 0; diff --git a/drivers/gpu/drm/xe/xe_vram_freq.c b/drivers/gpu/drm/xe/xe_vram_freq.c index b26e26d73dae..17bc84da4cdc 100644 --- a/drivers/gpu/drm/xe/xe_vram_freq.c +++ b/drivers/gpu/drm/xe/xe_vram_freq.c @@ -34,7 +34,7 @@ static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct xe_tile *tile = dev_to_tile(dev); - u32 val, mbox; + u32 val = 0, mbox; int err; mbox = REG_FIELD_PREP(PCODE_MB_COMMAND, PCODE_FREQUENCY_CONFIG) @@ -56,7 +56,7 @@ static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct xe_tile *tile = dev_to_tile(dev); - u32 val, mbox; + u32 val = 0, mbox; int err; mbox = REG_FIELD_PREP(PCODE_MB_COMMAND, PCODE_FREQUENCY_CONFIG) From efeb036ffd2a0d1c1afba3d74bd5965825d1a646 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:17:56 -0700 Subject: [PATCH 104/169] drm/xe/psmi: Add GuC flag to enable PSMI PSMI allows to capture data from the GPU useful for early validation. From the kernel side there isn't much to be done, just a few things: 1) Toggle the feature support in GuC 2) Enable some additional WAs 3) Allocate buffers Here is the first step, with the next ones to follow. For now everything is disabled through a check in configfs that is currently hardcoded to disabled. Cc: Matt Roper Cc: Daniele Ceraolo Spurio Cc: John Harrison Reviewed-by: Vinay Belgaumkar Link: https://lore.kernel.org/r/20250821-psmi-v5-1-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.h | 2 ++ drivers/gpu/drm/xe/xe_guc.c | 7 ++++++- drivers/gpu/drm/xe/xe_guc_fwif.h | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index fb8764008089..c14588b86e83 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -16,12 +16,14 @@ void xe_configfs_exit(void); bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); void xe_configfs_clear_survivability_mode(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); +static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } #else static inline int xe_configfs_init(void) { return 0; } static inline void xe_configfs_exit(void) { } static inline bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { return false; } static inline void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) { } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } +static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } #endif #endif diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 433abc787f7b..883f700e532a 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -16,6 +16,7 @@ #include "regs/xe_guc_regs.h" #include "regs/xe_irq_regs.h" #include "xe_bo.h" +#include "xe_configfs.h" #include "xe_device.h" #include "xe_force_wake.h" #include "xe_gt.h" @@ -81,11 +82,15 @@ static u32 guc_ctl_debug_flags(struct xe_guc *guc) static u32 guc_ctl_feature_flags(struct xe_guc *guc) { + struct xe_device *xe = guc_to_xe(guc); u32 flags = GUC_CTL_ENABLE_LITE_RESTORE; - if (!guc_to_xe(guc)->info.skip_guc_pc) + if (!xe->info.skip_guc_pc) flags |= GUC_CTL_ENABLE_SLPC; + if (xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev))) + flags |= GUC_CTL_ENABLE_PSMI_LOGGING; + return flags; } diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index ca9f999d38d1..400004f1838a 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -112,6 +112,7 @@ struct guc_update_exec_queue_policy { #define GUC_CTL_FEATURE 2 #define GUC_CTL_ENABLE_SLPC BIT(2) #define GUC_CTL_ENABLE_LITE_RESTORE BIT(4) +#define GUC_CTL_ENABLE_PSMI_LOGGING BIT(7) #define GUC_CTL_DISABLE_SCHEDULER BIT(14) #define GUC_CTL_DEBUG 3 From aaa0c1f50a3d3fc1624360eaeb7fe69af62f7107 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:17:57 -0700 Subject: [PATCH 105/169] drm/xe/psmi: Add debugfs interface for PSMI Requirement for PSMI capture is to have a physically contiguous buffer. All the needed configuration is done by the userspace tool directly to the GPU via mmio access. This interface only support allocating from VRAM regions. For integrated devices, the PSMI buffer is in SYSTEM memory and should be allocated by userspace using hugetlbfs. Here we add the ability to allocate a region of physically contiguous memory by writing to debugfs file (listed below). For multi-tile devices, the capture tool requires ability to allocate a capture buffer per tile (VRAM region) and so user can specify a region_mask. The tool then can mmap the buffers via direct mmap of the PCIBAR via sysfs. To support the capture tool, 3 new debugfs entries are added: psmi_capture_addr - physical address per VRAM region's capture buffer psmi_capture_region_mask - select which region(s) to allocate a buffer psmi_capture_size - size of current capture buffer Writing psmi_capture_size will allocate new buffer of requested size per region after freeing any current buffers. Cc: Matt Roper Cc: Vinay Belgaumkar Original-author: Brian Welty Reviewed-by: Vinay Belgaumkar # v2 Link: https://lore.kernel.org/r/20250821-psmi-v5-2-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_debugfs.c | 3 + drivers/gpu/drm/xe/xe_device.c | 5 + drivers/gpu/drm/xe/xe_device_types.h | 8 + drivers/gpu/drm/xe/xe_psmi.c | 306 +++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_psmi.h | 14 ++ 6 files changed, 337 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_psmi.c create mode 100644 drivers/gpu/drm/xe/xe_psmi.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 8e0c3412a757..85b8d3a59ef0 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -98,6 +98,7 @@ xe-y += xe_bb.o \ xe_pcode.o \ xe_pm.o \ xe_preempt_fence.o \ + xe_psmi.o \ xe_pt.o \ xe_pt_walk.o \ xe_pxp.o \ diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 0b4a532f7c45..bc717519502d 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -20,6 +20,7 @@ #include "xe_guc_ads.h" #include "xe_mmio.h" #include "xe_pm.h" +#include "xe_psmi.h" #include "xe_pxp_debugfs.h" #include "xe_sriov.h" #include "xe_sriov_pf.h" @@ -400,6 +401,8 @@ void xe_debugfs_register(struct xe_device *xe) xe_pxp_debugfs_register(xe->pxp); + xe_psmi_debugfs_register(xe); + fault_create_debugfs_attr("fail_gt_reset", root, >_reset_failure); if (IS_SRIOV_PF(xe)) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 3e0402dff423..d4e186c6545a 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -54,6 +54,7 @@ #include "xe_pcode.h" #include "xe_pm.h" #include "xe_pmu.h" +#include "xe_psmi.h" #include "xe_pxp.h" #include "xe_query.h" #include "xe_shrinker.h" @@ -908,6 +909,10 @@ int xe_device_probe(struct xe_device *xe) if (err) return err; + err = xe_psmi_init(xe); + if (err) + return err; + err = drm_dev_register(&xe->drm, 0); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 38c8329b4d2c..86506cdfc801 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -576,6 +576,14 @@ struct xe_device { atomic64_t global_total_pages; #endif + /** @psmi: GPU debugging via additional validation HW */ + struct { + /** @psmi.capture_obj: PSMI buffer for VRAM */ + struct xe_bo *capture_obj[XE_MAX_TILES_PER_DEVICE + 1]; + /** @psmi.region_mask: Mask of valid memory regions */ + u8 region_mask; + } psmi; + /* private: */ #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) diff --git a/drivers/gpu/drm/xe/xe_psmi.c b/drivers/gpu/drm/xe/xe_psmi.c new file mode 100644 index 000000000000..a2c9ff5bfd59 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_psmi.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_configfs.h" +#include "xe_psmi.h" + +/* + * PSMI capture support + * + * Requirement for PSMI capture is to have a physically contiguous buffer. The + * PSMI tool owns doing all necessary configuration (MMIO register writes are + * done from user-space). However, KMD needs to provide the PSMI tool with the + * required physical address of the base of PSMI buffer in case of VRAM. + * + * VRAM backed PSMI buffer: + * Buffer is allocated as GEM object and with XE_BO_CREATE_PINNED_BIT flag which + * creates a contiguous allocation. The physical address is returned from + * psmi_debugfs_capture_addr_show(). PSMI tool can mmap the buffer via the + * PCIBAR through sysfs. + * + * SYSTEM memory backed PSMI buffer: + * Interface here does not support allocating from SYSTEM memory region. The + * PSMI tool needs to allocate memory themselves using hugetlbfs. In order to + * get the physical address, user-space can query /proc/[pid]/pagemap. As an + * alternative, CMA debugfs could also be used to allocate reserved CMA memory. + */ + +static bool psmi_enabled(struct xe_device *xe) +{ + return xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)); +} + +static void psmi_free_object(struct xe_bo *bo) +{ + xe_bo_lock(bo, NULL); + xe_bo_unpin(bo); + xe_bo_unlock(bo); + xe_bo_put(bo); +} + +/* + * Free PSMI capture buffer objects. + */ +static void psmi_cleanup(struct xe_device *xe) +{ + unsigned long id, region_mask = xe->psmi.region_mask; + struct xe_bo *bo; + + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = xe->psmi.capture_obj[id]; + if (bo) { + psmi_free_object(bo); + xe->psmi.capture_obj[id] = NULL; + } + } +} + +static struct xe_bo *psmi_alloc_object(struct xe_device *xe, + unsigned int id, size_t bo_size) +{ + struct xe_bo *bo = NULL; + struct xe_tile *tile; + int err; + + if (!id || !bo_size) + return NULL; + + tile = &xe->tiles[id - 1]; + + /* VRAM: Allocate GEM object for the capture buffer */ + bo = xe_bo_create_locked(xe, tile, NULL, bo_size, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_PINNED | + XE_BO_FLAG_PINNED_LATE_RESTORE | + XE_BO_FLAG_NEEDS_CPU_ACCESS); + + if (!IS_ERR(bo)) { + /* Buffer written by HW, ensure stays resident */ + err = xe_bo_pin(bo); + if (err) + bo = ERR_PTR(err); + xe_bo_unlock(bo); + } + + return bo; +} + +/* + * Allocate PSMI capture buffer objects (via debugfs set function), based on + * which regions the user has selected in region_mask. @size: size in bytes + * (should be power of 2) + * + * Always release/free the current buffer objects before attempting to allocate + * new ones. Size == 0 will free all current buffers. + * + * Note, we don't write any registers as the capture tool is already configuring + * all PSMI registers itself via mmio space. + */ +static int psmi_resize_object(struct xe_device *xe, size_t size) +{ + unsigned long id, region_mask = xe->psmi.region_mask; + struct xe_bo *bo = NULL; + int err = 0; + + /* if resizing, free currently allocated buffers first */ + psmi_cleanup(xe); + + /* can set size to 0, in which case, now done */ + if (!size) + return 0; + + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = psmi_alloc_object(xe, id, size); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + break; + } + xe->psmi.capture_obj[id] = bo; + + drm_info(&xe->drm, + "PSMI capture size requested: %zu bytes, allocated: %lu:%zu\n", + size, id, bo ? xe_bo_size(bo) : 0); + } + + /* on error, reverse what was allocated */ + if (err) + psmi_cleanup(xe); + + return err; +} + +/* + * Returns an address for the capture tool to use to find start of capture + * buffer. Capture tool requires the capability to have a buffer allocated per + * each tile (VRAM region), thus we return an address for each region. + */ +static int psmi_debugfs_capture_addr_show(struct seq_file *m, void *data) +{ + struct xe_device *xe = m->private; + unsigned long id, region_mask; + struct xe_bo *bo; + u64 val; + + region_mask = xe->psmi.region_mask; + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + /* VRAM region */ + bo = xe->psmi.capture_obj[id]; + if (!bo) + continue; + + /* pinned, so don't need bo_lock */ + val = __xe_bo_addr(bo, 0, PAGE_SIZE); + seq_printf(m, "%ld: 0x%llx\n", id, val); + } + + return 0; +} + +/* + * Return capture buffer size, using the size from first allocated object that + * is found. This works because all objects must be of the same size. + */ +static int psmi_debugfs_capture_size_get(void *data, u64 *val) +{ + unsigned long id, region_mask; + struct xe_device *xe = data; + struct xe_bo *bo; + + region_mask = xe->psmi.region_mask; + for_each_set_bit(id, ®ion_mask, + ARRAY_SIZE(xe->psmi.capture_obj)) { + /* smem should never be set */ + xe_assert(xe, id); + + bo = xe->psmi.capture_obj[id]; + if (bo) { + *val = xe_bo_size(bo); + return 0; + } + } + + /* no capture objects are allocated */ + *val = 0; + + return 0; +} + +/* + * Set size of PSMI capture buffer. This triggers the allocation of capture + * buffer in each memory region as specified with prior write to + * psmi_capture_region_mask. + */ +static int psmi_debugfs_capture_size_set(void *data, u64 val) +{ + struct xe_device *xe = data; + + /* user must have specified at least one region */ + if (!xe->psmi.region_mask) + return -EINVAL; + + return psmi_resize_object(xe, val); +} + +static int psmi_debugfs_capture_region_mask_get(void *data, u64 *val) +{ + struct xe_device *xe = data; + + *val = xe->psmi.region_mask; + + return 0; +} + +/* + * Select VRAM regions for multi-tile devices, only allowed when buffer is not + * currently allocated. + */ +static int psmi_debugfs_capture_region_mask_set(void *data, u64 region_mask) +{ + struct xe_device *xe = data; + u64 size = 0; + + /* SMEM is not supported (see comments at top of file) */ + if (region_mask & 0x1) + return -EOPNOTSUPP; + + /* input bitmask should contain only valid TTM regions */ + if (!region_mask || region_mask & ~xe->info.mem_region_mask) + return -EINVAL; + + /* only allow setting mask if buffer is not yet allocated */ + psmi_debugfs_capture_size_get(xe, &size); + if (size) + return -EBUSY; + + xe->psmi.region_mask = region_mask; + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(psmi_debugfs_capture_addr); + +DEFINE_DEBUGFS_ATTRIBUTE(psmi_debugfs_capture_region_mask_fops, + psmi_debugfs_capture_region_mask_get, + psmi_debugfs_capture_region_mask_set, + "0x%llx\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(psmi_debugfs_capture_size_fops, + psmi_debugfs_capture_size_get, + psmi_debugfs_capture_size_set, + "%lld\n"); + +void xe_psmi_debugfs_register(struct xe_device *xe) +{ + struct drm_minor *minor; + + if (!psmi_enabled(xe)) + return; + + minor = xe->drm.primary; + if (!minor->debugfs_root) + return; + + debugfs_create_file("psmi_capture_addr", + 0400, minor->debugfs_root, xe, + &psmi_debugfs_capture_addr_fops); + + debugfs_create_file("psmi_capture_region_mask", + 0600, minor->debugfs_root, xe, + &psmi_debugfs_capture_region_mask_fops); + + debugfs_create_file("psmi_capture_size", + 0600, minor->debugfs_root, xe, + &psmi_debugfs_capture_size_fops); +} + +static void psmi_fini(void *arg) +{ + psmi_cleanup(arg); +} + +int xe_psmi_init(struct xe_device *xe) +{ + if (!psmi_enabled(xe)) + return 0; + + return devm_add_action(xe->drm.dev, psmi_fini, xe); +} diff --git a/drivers/gpu/drm/xe/xe_psmi.h b/drivers/gpu/drm/xe/xe_psmi.h new file mode 100644 index 000000000000..b1dfba80d893 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_psmi.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PSMI_H_ +#define _XE_PSMI_H_ + +struct xe_device; + +int xe_psmi_init(struct xe_device *xe); +void xe_psmi_debugfs_register(struct xe_device *xe); + +#endif From d67b1dfad00d7c28155a434b2a8a0012adc1b458 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:17:58 -0700 Subject: [PATCH 106/169] drm/xe/rtp: Add match for psmi Add match to be used on WAs for only enabling workarounds if psmi is intended to be used. Reviewed-by: Vinay Belgaumkar Link: https://lore.kernel.org/r/20250821-psmi-v5-3-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_rtp.c | 7 +++++++ drivers/gpu/drm/xe/xe_rtp.h | 3 +++ 2 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c index 95571b87aa73..47ea1521dc80 100644 --- a/drivers/gpu/drm/xe/xe_rtp.c +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -9,6 +9,7 @@ #include +#include "xe_configfs.h" #include "xe_gt.h" #include "xe_gt_topology.h" #include "xe_macros.h" @@ -363,3 +364,9 @@ bool xe_rtp_match_not_sriov_vf(const struct xe_gt *gt, { return !IS_SRIOV_VF(gt_to_xe(gt)); } + +bool xe_rtp_match_psmi_enabled(const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_configfs_get_psmi_enabled(to_pci_dev(gt_to_xe(gt)->drm.dev)); +} diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h index 5ed6c14b9ae3..7951fefdbe04 100644 --- a/drivers/gpu/drm/xe/xe_rtp.h +++ b/drivers/gpu/drm/xe/xe_rtp.h @@ -477,4 +477,7 @@ bool xe_rtp_match_first_render_or_compute(const struct xe_gt *gt, bool xe_rtp_match_not_sriov_vf(const struct xe_gt *gt, const struct xe_hw_engine *hwe); +bool xe_rtp_match_psmi_enabled(const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + #endif From 29042df3acdc7364af1c251b2a05f7c1c8fe0401 Mon Sep 17 00:00:00 2001 From: Badal Nilawar Date: Thu, 21 Aug 2025 09:17:59 -0700 Subject: [PATCH 107/169] drm/xe/psmi: Add Wa_14020001231 Enable Wa 14020001231 to block psmi interrupts during C6 entry exit flow. It's only enabled if PSMI is enabled in runtime. Signed-off-by: Badal Nilawar Reviewed-by: Matt Atwood Link: https://lore.kernel.org/r/20250821-psmi-v5-4-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/abi/guc_klvs_abi.h | 1 + drivers/gpu/drm/xe/xe_guc_ads.c | 4 ++++ drivers/gpu/drm/xe/xe_wa_oob.rules | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index 31dbfeee289e..0e78351c6ef5 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -390,6 +390,7 @@ enum { */ enum xe_guc_klv_ids { GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED = 0x9002, + GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT = 0x9004, GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING = 0x9005, GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE = 0x9007, GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE = 0x9008, diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index d7da67637079..5631722f34f5 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -359,6 +359,10 @@ static void guc_waklv_init(struct xe_guc_ads *ads) GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG); } + if (XE_GT_WA(gt, 14020001231)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); + size = guc_ads_waklv_size(ads) - remain; if (!size) return; diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 8d0aabab6777..303a5e05d993 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -68,6 +68,10 @@ no_media_l3 MEDIA_VERSION(3000) MEDIA_VERSION_RANGE(1300, 3000) MEDIA_VERSION(3002) GRAPHICS_VERSION(3003) +14020001231 GRAPHICS_VERSION_RANGE(2001,2004), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION(3000), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION(3002), FUNC(xe_rtp_match_psmi_enabled) # SoC workaround - currently applies to all platforms with the following # primary GT GMDID From 95b3899b4d25a49abf830ee1bc97940a7c1c7e97 Mon Sep 17 00:00:00 2001 From: Vinay Belgaumkar Date: Thu, 21 Aug 2025 09:18:00 -0700 Subject: [PATCH 108/169] drm/xe/psmi: Add Wa_16023683509 This WA ensures GuC will restore the media MCFG registers at C6 exit. Signed-off-by: Vinay Belgaumkar Reviewed-by: Matt Atwood Link: https://lore.kernel.org/r/20250821-psmi-v5-5-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_guc.c | 3 +++ drivers/gpu/drm/xe/xe_guc_fwif.h | 1 + drivers/gpu/drm/xe/xe_wa_oob.rules | 2 ++ 3 files changed, 6 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 883f700e532a..37d06c51180c 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -219,6 +219,9 @@ static u32 guc_ctl_wa_flags(struct xe_guc *guc) if (XE_GT_WA(gt, 14018913170)) flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6; + if (XE_GT_WA(gt, 16023683509)) + flags |= GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6; + return flags; } diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index 400004f1838a..0508f1064178 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -108,6 +108,7 @@ struct guc_update_exec_queue_policy { #define GUC_WA_RENDER_RST_RC6_EXIT BIT(19) #define GUC_WA_RCS_REGS_IN_CCS_REGS_LIST BIT(21) #define GUC_WA_ENABLE_TSC_CHECK_ON_RC6 BIT(22) +#define GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6 BIT(25) #define GUC_CTL_FEATURE 2 #define GUC_CTL_ENABLE_SLPC BIT(2) diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 303a5e05d993..27cd19b98f8e 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -72,6 +72,8 @@ no_media_l3 MEDIA_VERSION(3000) MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) MEDIA_VERSION(3000), FUNC(xe_rtp_match_psmi_enabled) MEDIA_VERSION(3002), FUNC(xe_rtp_match_psmi_enabled) +16023683509 MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) + MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled) # SoC workaround - currently applies to all platforms with the following # primary GT GMDID From 49245b49614870a0a135468be1bd0a6e0006a128 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:01 -0700 Subject: [PATCH 109/169] drm/xe/configfs: Simplify kernel doc From the caller perspective reading the documentation, there's no need to be so specific about everything the function is doing/checking. Just document the functionality a caller cares about. Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250821-psmi-v5-6-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 853da2ee837a..17b1d6ae1ff6 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -389,10 +389,7 @@ static struct xe_config_group_device *find_xe_config_group_device(struct pci_dev * xe_configfs_get_survivability_mode - get configfs survivability mode attribute * @pdev: pci device * - * find the configfs group that belongs to the pci device and return - * the survivability mode attribute - * - * Return: survivability mode if config group is found, false otherwise + * Return: survivability_mode attribute in configfs */ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { @@ -409,11 +406,8 @@ bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) } /** - * xe_configfs_clear_survivability_mode - clear configfs survivability mode attribute + * xe_configfs_clear_survivability_mode - clear configfs survivability mode * @pdev: pci device - * - * find the configfs group that belongs to the pci device and clear survivability - * mode attribute */ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) { @@ -433,10 +427,7 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) * xe_configfs_get_engines_allowed - get engine allowed mask from configfs * @pdev: pci device * - * Find the configfs group that belongs to the pci device and return - * the mask of engines allowed to be used. - * - * Return: engine mask with allowed engines + * Return: engine mask with allowed engines set in configfs */ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { From afe902848b417da81f60b3d15cbae40a67286dbf Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:02 -0700 Subject: [PATCH 110/169] drm/xe/configfs: Allow to enable PSMI Now that additional WAs are in place and it's possible to allocate buffers through debugfs, add the configfs attribute to turn PSMI on. Cc: Matt Roper Cc: Daniele Ceraolo Spurio Cc: John Harrison Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250821-psmi-v5-7-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 66 ++++++++++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_configfs.h | 2 +- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 17b1d6ae1ff6..61376dfc74bb 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -77,6 +77,16 @@ * available for migrations, but it's disabled. This is intended for debugging * purposes only. * + * PSMI + * ---- + * + * Enable extra debugging capabilities to trace engine execution. Only useful + * during early platform enabling and requires additional hardware connected. + * Once it's enabled, additionals WAs are added and runtime configuration is + * done via debugfs. Example to enable it:: + * + * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/enable_psmi + * * Remove devices * ============== * @@ -89,8 +99,9 @@ struct xe_config_group_device { struct config_group group; struct xe_config_device { - bool survivability_mode; u64 engines_allowed; + bool survivability_mode; + bool enable_psmi; } config; /* protects attributes */ @@ -98,8 +109,9 @@ struct xe_config_group_device { }; static const struct xe_config_device device_defaults = { - .survivability_mode = false, .engines_allowed = U64_MAX, + .survivability_mode = false, + .enable_psmi = false, }; static void set_device_defaults(struct xe_config_device *config) @@ -243,12 +255,38 @@ static ssize_t engines_allowed_store(struct config_item *item, const char *page, return len; } -CONFIGFS_ATTR(, survivability_mode); +static ssize_t enable_psmi_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%d\n", dev->enable_psmi); +} + +static ssize_t enable_psmi_store(struct config_item *item, const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + bool val; + int ret; + + ret = kstrtobool(page, &val); + if (ret) + return ret; + + mutex_lock(&dev->lock); + dev->config.enable_psmi = val; + mutex_unlock(&dev->lock); + + return len; +} + +CONFIGFS_ATTR(, enable_psmi); CONFIGFS_ATTR(, engines_allowed); +CONFIGFS_ATTR(, survivability_mode); static struct configfs_attribute *xe_config_device_attrs[] = { - &attr_survivability_mode, + &attr_enable_psmi, &attr_engines_allowed, + &attr_survivability_mode, NULL, }; @@ -443,6 +481,26 @@ u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) return engines_allowed; } +/** + * xe_configfs_get_psmi_enabled - get configfs enable_psmi setting + * @pdev: pci device + * + * Return: enable_psmi setting in configfs + */ +bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool ret; + + if (!dev) + return false; + + ret = dev->config.enable_psmi; + config_item_put(&dev->group.cg_item); + + return ret; +} + int __init xe_configfs_init(void) { int ret; diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index c14588b86e83..603dd7796c8b 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -16,7 +16,7 @@ void xe_configfs_exit(void); bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); void xe_configfs_clear_survivability_mode(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); -static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } +bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); #else static inline int xe_configfs_init(void) { return 0; } static inline void xe_configfs_exit(void) { } From 734197a933cd2fb4adac7213fd1fc4df284cae14 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:03 -0700 Subject: [PATCH 111/169] drm/xe/configfs: Use guard() for dev->lock Instead of the manual lock()/unlock() pattern, use guard() which will make things easier for handling errors or early returns. Reviewed-by: Dnyaneshwar Bhadane Link: https://lore.kernel.org/r/20250821-psmi-v5-8-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 61376dfc74bb..de73c4ff8f4f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -164,9 +165,8 @@ static ssize_t survivability_mode_store(struct config_item *item, const char *pa if (ret) return ret; - mutex_lock(&dev->lock); + guard(mutex)(&dev->lock); dev->config.survivability_mode = survivability_mode; - mutex_unlock(&dev->lock); return len; } @@ -248,9 +248,8 @@ static ssize_t engines_allowed_store(struct config_item *item, const char *page, val |= mask; } - mutex_lock(&dev->lock); + guard(mutex)(&dev->lock); dev->config.engines_allowed = val; - mutex_unlock(&dev->lock); return len; } @@ -272,9 +271,8 @@ static ssize_t enable_psmi_store(struct config_item *item, const char *page, siz if (ret) return ret; - mutex_lock(&dev->lock); + guard(mutex)(&dev->lock); dev->config.enable_psmi = val; - mutex_unlock(&dev->lock); return len; } @@ -454,9 +452,8 @@ void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) if (!dev) return; - mutex_lock(&dev->lock); + guard(mutex)(&dev->lock); dev->config.survivability_mode = 0; - mutex_unlock(&dev->lock); config_group_put(&dev->group); } From 3eb2280f6aa424bc99d59ff91ca741d6e240caee Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:04 -0700 Subject: [PATCH 112/169] drm/xe/configfs: Use tree-like output in documentation When documenting the directories, use an output similar to the `tree` command and add VFs and missing attributes. Reviewed-by: Dnyaneshwar Bhadane Link: https://lore.kernel.org/r/20250821-psmi-v5-9-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index de73c4ff8f4f..25b47106413b 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -39,8 +39,16 @@ * used to configure it:: * * /sys/kernel/config/xe/ - * .. 0000:03:00.0/ - * ... survivability_mode + * ├── 0000:00:02.0 + * │   └── ... + * ├── 0000:00:02.1 + * │   └── ... + * : + * └── 0000:03:00.0 + * ├── survivability_mode + * ├── engines_allowed + * └── enable_psmi + * * * Configure Attributes * ==================== From e2b33fce5eb08b37e38188b219a32c105cdc606e Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:05 -0700 Subject: [PATCH 113/169] drm/xe/configfs: Improve documentation steps The steps are roughly: 1. Load the module without binding to the device 2. Configure the desired device 3. Bind the device Move the binding part to the "Create devices" since it's not exclusive to the survivability_mode attribute and better document the steps. Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250821-psmi-v5-10-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 25b47106413b..089e21fefcfd 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -29,11 +29,18 @@ * See Documentation/filesystems/configfs.rst for more information about how configfs works. * * Create devices - * =============== + * ============== * - * In order to create a device, the user has to create a directory inside ``'xe'``:: + * To create a device, the ``xe`` module should already be loaded, but some + * attributes can only be set before binding the device. It can be accomplished + * by blocking the driver autoprobe: * - * mkdir /sys/kernel/config/xe/0000:03:00.0/ + * # echo 0 > /sys/bus/pci/drivers_autoprobe + * # modprobe xe + * + * In order to create a device, the user has to create a directory inside ``xe``:: + * + * # mkdir /sys/kernel/config/xe/0000:03:00.0/ * * Every device created is populated by the driver with entries that can be * used to configure it:: @@ -49,6 +56,12 @@ * ├── engines_allowed * └── enable_psmi * + * After configuring the attributes as per next section, the device can be + * probed with:: + * + * # echo 0000:03:00.0 > /sys/bus/pci/drivers/xe/bind + * # # or + * # echo 0000:03:00.0 > /sys/bus/pci/drivers_probe * * Configure Attributes * ==================== @@ -60,7 +73,6 @@ * effect when probing the device. Example to enable it:: * * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/survivability_mode - * # echo 0000:03:00.0 > /sys/bus/pci/drivers/xe/bind (Enters survivability mode if supported) * * Allowed engines: * ---------------- From 66b21c338e6f05a980fae76c221d7aea4f1538c2 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:06 -0700 Subject: [PATCH 114/169] drm/xe/configfs: Minor fixes to documentation Add a few missing punctuation and line breaks and make the syntax for code snippets common to all of them. Reviewed-by: Dnyaneshwar Bhadane Link: https://lore.kernel.org/r/20250821-psmi-v5-11-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 089e21fefcfd..26c5e1b29263 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -24,9 +24,10 @@ * ========= * * Configfs is a filesystem-based manager of kernel objects. XE KMD registers a - * configfs subsystem called ``'xe'`` that creates a directory in the mounted configfs directory - * The user can create devices under this directory and configure them as necessary - * See Documentation/filesystems/configfs.rst for more information about how configfs works. + * configfs subsystem called ``xe`` that creates a directory in the mounted + * configfs directory. The user can create devices under this directory and + * configure them as necessary. See Documentation/filesystems/configfs.rst for + * more information about how configfs works. * * Create devices * ============== @@ -113,7 +114,7 @@ * * The created device directories can be removed using ``rmdir``:: * - * rmdir /sys/kernel/config/xe/0000:03:00.0/ + * # rmdir /sys/kernel/config/xe/0000:03:00.0/ */ struct xe_config_group_device { From 13dda74a164b570d93d412e812dc80549506410f Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Thu, 21 Aug 2025 09:18:07 -0700 Subject: [PATCH 115/169] drm/xe/configfs: Dump custom settings when binding Device configuration using configfs could be prepared long time prior the driver load. Currently all the xe configfs entries are for things that are important to have in the log if a non-default value is being used. Add a info-level message about that with the individual entries that are different than the default. Based on previous patch by Michal Wajdeczko. Cc: Michal Wajdeczko Cc: John Harrison Reviewed-by: Vinay Belgaumkar Link: https://lore.kernel.org/r/20250821-psmi-v5-12-34ab7550d3d8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 39 ++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_configfs.h | 2 ++ drivers/gpu/drm/xe/xe_pci.c | 3 +++ 3 files changed, 44 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 26c5e1b29263..29a12d02443d 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -442,6 +442,45 @@ static struct xe_config_group_device *find_xe_config_group_device(struct pci_dev return to_xe_config_group_device(item); } +static void dump_custom_dev_config(struct pci_dev *pdev, + struct xe_config_group_device *dev) +{ +#define PRI_CUSTOM_ATTR(fmt_, attr_) do { \ + if (dev->config.attr_ != device_defaults.attr_) \ + pci_info(pdev, "configfs: " __stringify(attr_) " = " fmt_ "\n", \ + dev->config.attr_); \ + } while (0) + + PRI_CUSTOM_ATTR("%llx", engines_allowed); + PRI_CUSTOM_ATTR("%d", enable_psmi); + PRI_CUSTOM_ATTR("%d", survivability_mode); + +#undef PRI_CUSTOM_ATTR +} + +/** + * xe_configfs_check_device() - Test if device was configured by configfs + * @pdev: the &pci_dev device to test + * + * Try to find the configfs group that belongs to the specified pci device + * and print a diagnostic message if different than the default value. + */ +void xe_configfs_check_device(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + + if (!dev) + return; + + /* memcmp here is safe as both are zero-initialized */ + if (memcmp(&dev->config, &device_defaults, sizeof(dev->config))) { + pci_info(pdev, "Found custom settings in configfs\n"); + dump_custom_dev_config(pdev, dev); + } + + config_group_put(&dev->group); +} + /** * xe_configfs_get_survivability_mode - get configfs survivability mode attribute * @pdev: pci device diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index 603dd7796c8b..58c8c3164000 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -13,6 +13,7 @@ struct pci_dev; #if IS_ENABLED(CONFIG_CONFIGFS_FS) int xe_configfs_init(void); void xe_configfs_exit(void); +void xe_configfs_check_device(struct pci_dev *pdev); bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); void xe_configfs_clear_survivability_mode(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); @@ -20,6 +21,7 @@ bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); #else static inline int xe_configfs_init(void) { return 0; } static inline void xe_configfs_exit(void) { } +static inline void xe_configfs_check_device(struct pci_dev *pdev) { } static inline bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { return false; } static inline void xe_configfs_clear_survivability_mode(struct pci_dev *pdev) { } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 419e29cebbba..a8bab19111c3 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -18,6 +18,7 @@ #include "display/xe_display.h" #include "regs/xe_gt_regs.h" #include "regs/xe_regs.h" +#include "xe_configfs.h" #include "xe_device.h" #include "xe_drv.h" #include "xe_gt.h" @@ -816,6 +817,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct xe_device *xe; int err; + xe_configfs_check_device(pdev); + if (desc->require_force_probe && !id_forced(pdev->device)) { dev_info(&pdev->dev, "Your graphics device %04x is not officially supported\n" From 8ae04fe9ffc93d6bc3bc63ac08375427d69cee06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20Kempczy=C5=84ski?= Date: Wed, 20 Aug 2025 10:39:04 +0200 Subject: [PATCH 116/169] drm/xe/xe_sync: avoid race during ufence signaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Marking ufence as signalled after copy_to_user() is too late. Worker thread which signals ufence by memory write might be raced with another userspace vm-bind call. In map/unmap scenario unmap may still see ufence is not signalled causing -EBUSY. Change the order of marking / write to user-fence fixes this issue. Fixes: 977e5b82e090 ("drm/xe: Expose user fence from xe_sync_entry") Link: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/5536 Signed-off-by: Zbigniew Kempczyński Cc: Matthew Brost Cc: Matthew Auld Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250820083903.2109891-2-zbigniew.kempczynski@intel.com --- drivers/gpu/drm/xe/xe_sync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index f87276df18f2..82872a51f098 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -77,6 +77,7 @@ static void user_fence_worker(struct work_struct *w) { struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); + WRITE_ONCE(ufence->signalled, 1); if (mmget_not_zero(ufence->mm)) { kthread_use_mm(ufence->mm); if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) @@ -91,7 +92,6 @@ static void user_fence_worker(struct work_struct *w) * Wake up waiters only after updating the ufence state, allowing the UMD * to safely reuse the same ufence without encountering -EBUSY errors. */ - WRITE_ONCE(ufence->signalled, 1); wake_up_all(&ufence->xe->ufence_wq); user_fence_put(ufence); } From 0a51bf3e54dd8b77e6f1febbbb66def0660862d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 21 Aug 2025 16:30:43 +0200 Subject: [PATCH 117/169] drm/xe/vm: Don't pin the vm_resv during validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinning has the odd side-effect that unlocking *any* resv during validation triggers an "unlocking pinned lock" warning. Cc: Matthew Brost Fixes: 9d5558649f68 ("drm/xe: Rework eviction rejection of bound external bos") Signed-off-by: Thomas Hellström Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821143045.106005-2-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_bo.c | 5 ++--- drivers/gpu/drm/xe/xe_vm.h | 15 ++------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 6fea39842e1e..11eaf3b06766 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -2468,7 +2468,6 @@ int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict) .no_wait_gpu = false, .gfp_retry_mayfail = true, }; - struct pin_cookie cookie; int ret; if (vm) { @@ -2479,10 +2478,10 @@ int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict) ctx.resv = xe_vm_resv(vm); } - cookie = xe_vm_set_validating(vm, allow_res_evict); + xe_vm_set_validating(vm, allow_res_evict); trace_xe_bo_validate(bo); ret = ttm_bo_validate(&bo->ttm, &bo->placement, &ctx); - xe_vm_clear_validating(vm, allow_res_evict, cookie); + xe_vm_clear_validating(vm, allow_res_evict); return ret; } diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 2f213737c7e5..2ecb417c19a2 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -315,22 +315,14 @@ void xe_vm_snapshot_free(struct xe_vm_snapshot *snap); * Register this task as currently making bos resident for the vm. Intended * to avoid eviction by the same task of shared bos bound to the vm. * Call with the vm's resv lock held. - * - * Return: A pin cookie that should be used for xe_vm_clear_validating(). */ -static inline struct pin_cookie xe_vm_set_validating(struct xe_vm *vm, - bool allow_res_evict) +static inline void xe_vm_set_validating(struct xe_vm *vm, bool allow_res_evict) { - struct pin_cookie cookie = {}; - if (vm && !allow_res_evict) { xe_vm_assert_held(vm); - cookie = lockdep_pin_lock(&xe_vm_resv(vm)->lock.base); /* Pairs with READ_ONCE in xe_vm_is_validating() */ WRITE_ONCE(vm->validating, current); } - - return cookie; } /** @@ -338,17 +330,14 @@ static inline struct pin_cookie xe_vm_set_validating(struct xe_vm *vm, * @vm: Pointer to the vm or NULL * @allow_res_evict: Eviction from @vm was allowed. Must be set to the same * value as for xe_vm_set_validation(). - * @cookie: Cookie obtained from xe_vm_set_validating(). * * Register this task as currently making bos resident for the vm. Intended * to avoid eviction by the same task of shared bos bound to the vm. * Call with the vm's resv lock held. */ -static inline void xe_vm_clear_validating(struct xe_vm *vm, bool allow_res_evict, - struct pin_cookie cookie) +static inline void xe_vm_clear_validating(struct xe_vm *vm, bool allow_res_evict) { if (vm && !allow_res_evict) { - lockdep_unpin_lock(&xe_vm_resv(vm)->lock.base, cookie); /* Pairs with READ_ONCE in xe_vm_is_validating() */ WRITE_ONCE(vm->validating, NULL); } From b5dd1505a31c5cbe62ec50a70e9c9a7bd2a8f703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 21 Aug 2025 16:30:44 +0200 Subject: [PATCH 118/169] drm/xe/tests/xe_dma_buf: Set the drm_object::dma_buf member MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This member is set when exporting using prime. However the xe_gem_prime_export() alone doesn't set it, since it's done later in the prime export flow. For the test, set it manually and remove the hack that set it temporarily when it was really needed. Signed-off-by: Thomas Hellström Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821143045.106005-3-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/tests/xe_dma_buf.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index c53f67ce4b0a..cde9530bef8c 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -57,16 +57,12 @@ static void check_residency(struct kunit *test, struct xe_bo *exported, return; /* - * Evict exporter. Note that the gem object dma_buf member isn't - * set from xe_gem_prime_export(), and it's needed for the move_notify() - * functionality, so hack that up here. Evicting the exported bo will + * Evict exporter. Evicting the exported bo will * evict also the imported bo through the move_notify() functionality if * importer is on a different device. If they're on the same device, * the exporter and the importer should be the same bo. */ - swap(exported->ttm.base.dma_buf, dmabuf); ret = xe_bo_evict(exported); - swap(exported->ttm.base.dma_buf, dmabuf); if (ret) { if (ret != -EINTR && ret != -ERESTARTSYS) KUNIT_FAIL(test, "Evicting exporter failed with err=%d.\n", @@ -139,6 +135,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) PTR_ERR(dmabuf)); goto out; } + bo->ttm.base.dma_buf = dmabuf; import = xe_gem_prime_import(&xe->drm, dmabuf); if (!IS_ERR(import)) { @@ -186,6 +183,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) KUNIT_FAIL(test, "dynamic p2p attachment failed with err=%ld\n", PTR_ERR(import)); } + bo->ttm.base.dma_buf = NULL; dma_buf_put(dmabuf); out: drm_gem_object_put(&bo->ttm.base); From 358ee50ab565f3c8ea32480e9d03127a81ba32f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 21 Aug 2025 16:30:45 +0200 Subject: [PATCH 119/169] drm/xe/vm: Clear the scratch_pt pointer on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid triggering a dereference of an error pointer on cleanup in xe_vm_free_scratch() by clearing any scratch_pt error pointer. Signed-off-by: Thomas Hellström Fixes: 06951c2ee72d ("drm/xe: Use NULL PTEs as scratch PTEs") Cc: Brian Welty Cc: Rodrigo Vivi Cc: Lucas De Marchi Cc: # v6.8+ Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821143045.106005-4-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_vm.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c86337e08a55..d3f6dc6b1779 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1635,8 +1635,12 @@ static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i); - if (IS_ERR(vm->scratch_pt[id][i])) - return PTR_ERR(vm->scratch_pt[id][i]); + if (IS_ERR(vm->scratch_pt[id][i])) { + int err = PTR_ERR(vm->scratch_pt[id][i]); + + vm->scratch_pt[id][i] = NULL; + return err; + } xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); } From ffdf968762e4fb3cdae54e811ec3525e67440a60 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Mon, 25 Aug 2025 08:28:41 -0700 Subject: [PATCH 120/169] drm/xe: Don't trigger rebind on initial dma-buf validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the first validate of an imported dma-buf (initial bind), the device has no GPU mappings, so a rebind is unnecessary. Rebinding here is harmful in multi-GPU setups and for VMs using preempt-fence mode, as it would evict in-flight GPU work. v2: - Drop dma_buf_validated, check for XE_PL_SYSTEM (Thomas) Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://lore.kernel.org/r/20250825152841.3837378-1-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 11eaf3b06766..0a6a32cfb771 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -816,7 +816,8 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, } if (ttm_bo->type == ttm_bo_type_sg) { - ret = xe_bo_move_notify(bo, ctx); + if (new_mem->mem_type == XE_PL_SYSTEM) + ret = xe_bo_move_notify(bo, ctx); if (!ret) ret = xe_bo_move_dmabuf(ttm_bo, new_mem); return ret; From 41be792f5baaf90d744a9a9e82994ce560ca9582 Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Mon, 25 Aug 2025 15:57:42 +0000 Subject: [PATCH 121/169] drm/xe: switch to local xbasename() helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit b0a2ee5567ab ("drm/xe: prepare xe_gen_wa_oob to be multi-use") introduced a call to basename(). The GNU version of this function is not portable and fails to build with alternative libc implementations like musl or bionic. This causes the following build error: drivers/gpu/drm/xe/xe_gen_wa_oob.c:130:12: error: assignment to ‘const char *’ from ‘int’ makes pointer from integer without a cast [-Wint-conversion] 130 | fn = basename(fn); | ^ While a POSIX version of basename() could be used, it would require a separate header plus the behavior differs from GNU version in that it might modify its argument. Not great. Instead, implement a local xbasename() helper based on strrchr() that provides the same functionality and avoids portability issues. Fixes: b0a2ee5567ab ("drm/xe: prepare xe_gen_wa_oob to be multi-use") Suggested-by: Lucas De Marchi Reviewed-by: Tiffany Yang Signed-off-by: Carlos Llamas Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250825155743.1132433-1-cmllamas@google.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_gen_wa_oob.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gen_wa_oob.c b/drivers/gpu/drm/xe/xe_gen_wa_oob.c index 6581cb0f0e59..247e41c1c48d 100644 --- a/drivers/gpu/drm/xe/xe_gen_wa_oob.c +++ b/drivers/gpu/drm/xe/xe_gen_wa_oob.c @@ -123,11 +123,19 @@ static int parse(FILE *input, FILE *csource, FILE *cheader, char *prefix) return 0; } +/* Avoid GNU vs POSIX basename() discrepancy, just use our own */ +static const char *xbasename(const char *s) +{ + const char *p = strrchr(s, '/'); + + return p ? p + 1 : s; +} + static int fn_to_prefix(const char *fn, char *prefix, size_t size) { size_t len; - fn = basename(fn); + fn = xbasename(fn); len = strlen(fn); if (len > size - 1) From 231bb0ee7aa5d1f0d077d3a30663f8ffd6860fa3 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:45 +0530 Subject: [PATCH 122/169] drm/xe/uapi: Add madvise interface This commit introduces a new madvise interface to support driver-specific ioctl operations. The madvise interface allows for more efficient memory management by providing hints to the driver about the expected memory usage and pte update policy for gpuvma. v2 (Matthew/Thomas) - Drop num_ops support - Drop purgeable support - Add kernel-docs - IOWR/IOW v3 (Matthew/Thomas) - Reorder attributes - use __u16 for migration_policy - use __u64 for reserved in unions - Avoid usage of vma Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-2-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- include/uapi/drm/xe_drm.h | 130 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index c721e130c1d2..4e6e9a9164ee 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -81,6 +81,7 @@ extern "C" { * - &DRM_IOCTL_XE_EXEC * - &DRM_IOCTL_XE_WAIT_USER_FENCE * - &DRM_IOCTL_XE_OBSERVATION + * - &DRM_IOCTL_XE_MADVISE */ /* @@ -102,6 +103,7 @@ extern "C" { #define DRM_XE_EXEC 0x09 #define DRM_XE_WAIT_USER_FENCE 0x0a #define DRM_XE_OBSERVATION 0x0b +#define DRM_XE_MADVISE 0x0c /* Must be kept compact -- no holes */ @@ -117,6 +119,7 @@ extern "C" { #define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param) +#define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise) /** * DOC: Xe IOCTL Extensions @@ -1978,6 +1981,133 @@ struct drm_xe_query_eu_stall { __u64 sampling_rates[]; }; +/** + * struct drm_xe_madvise - Input of &DRM_IOCTL_XE_MADVISE + * + * This structure is used to set memory attributes for a virtual address range + * in a VM. The type of attribute is specified by @type, and the corresponding + * union member is used to provide additional parameters for @type. + * + * Supported attribute types: + * - DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: Set preferred memory location. + * - DRM_XE_MEM_RANGE_ATTR_ATOMIC: Set atomic access policy. + * - DRM_XE_MEM_RANGE_ATTR_PAT: Set page attribute table index. + * + * Example: + * + * .. code-block:: C + * + * struct drm_xe_madvise madvise = { + *          .vm_id = vm_id, + *          .start = 0x100000, + *          .range = 0x2000, + *          .type = DRM_XE_MEM_RANGE_ATTR_ATOMIC, + *         .atomic_val = DRM_XE_ATOMIC_DEVICE, + * }; + * + * ioctl(fd, DRM_IOCTL_XE_MADVISE, &madvise); + * + */ +struct drm_xe_madvise { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @start: start of the virtual address range */ + __u64 start; + + /** @range: size of the virtual address range */ + __u64 range; + + /** @vm_id: vm_id of the virtual range */ + __u32 vm_id; + +#define DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC 0 +#define DRM_XE_MEM_RANGE_ATTR_ATOMIC 1 +#define DRM_XE_MEM_RANGE_ATTR_PAT 2 + /** @type: type of attribute */ + __u32 type; + + union { + /** + * @preferred_mem_loc: preferred memory location + * + * Used when @type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC + * + * Supported values for @preferred_mem_loc.devmem_fd: + * - DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE: set vram of faulting tile as preferred loc + * - DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM: set smem as preferred loc + * + * Supported values for @preferred_mem_loc.migration_policy: + * - DRM_XE_MIGRATE_ALL_PAGES + * - DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES + */ + struct { +#define DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE 0 +#define DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM -1 + /** @preferred_mem_loc.devmem_fd: fd for preferred loc */ + __u32 devmem_fd; + +#define DRM_XE_MIGRATE_ALL_PAGES 0 +#define DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES 1 + /** @preferred_mem_loc.migration_policy: Page migration policy */ + __u16 migration_policy; + + /** @preferred_mem_loc.pad : MBZ */ + __u16 pad; + + /** @preferred_mem_loc.reserved : Reserved */ + __u64 reserved; + } preferred_mem_loc; + + /** + * @atomic: Atomic access policy + * + * Used when @type == DRM_XE_MEM_RANGE_ATTR_ATOMIC. + * + * Supported values for @atomic.val: + * - DRM_XE_ATOMIC_UNDEFINED: Undefined or default behaviour + * Support both GPU and CPU atomic operations for system allocator + * Support GPU atomic operations for normal(bo) allocator + * - DRM_XE_ATOMIC_DEVICE: Support GPU atomic operations + * - DRM_XE_ATOMIC_GLOBAL: Support both GPU and CPU atomic operations + * - DRM_XE_ATOMIC_CPU: Support CPU atomic + */ + struct { +#define DRM_XE_ATOMIC_UNDEFINED 0 +#define DRM_XE_ATOMIC_DEVICE 1 +#define DRM_XE_ATOMIC_GLOBAL 2 +#define DRM_XE_ATOMIC_CPU 3 + /** @atomic.val: value of atomic operation */ + __u32 val; + + /** @atomic.pad: MBZ */ + __u32 pad; + + /** @atomic.reserved: Reserved */ + __u64 reserved; + } atomic; + + /** + * @pat_index: Page attribute table index + * + * Used when @type == DRM_XE_MEM_RANGE_ATTR_PAT. + */ + struct { + /** @pat_index.val: PAT index value */ + __u32 val; + + /** @pat_index.pad: MBZ */ + __u32 pad; + + /** @pat_index.reserved: Reserved */ + __u64 reserved; + } pat_index; + }; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + #if defined(__cplusplus) } #endif From 99a89e4e2d0c4e806af3dea4538c1fc9b208df3b Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:46 +0530 Subject: [PATCH 123/169] drm/xe/vm: Add attributes struct as member of vma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The attribute of xe_vma will determine the migration policy and the encoding of the page table entries (PTEs) for that vma. This attribute helps manage how memory pages are moved and how their addresses are translated. It will be used by madvise to set the behavior of the vma. v2 (Matthew Brost) - Add docs v3 (Matthew Brost) - Add uapi references - 80 characters line wrap Cc: Matthew Brost Reviewed-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://lore.kernel.org/r/20250821173104.3030148-3-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm_types.h | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 8a07feef503b..8e8138c2b80a 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -77,6 +77,33 @@ struct xe_userptr { #endif }; +/** + * struct xe_vma_mem_attr - memory attributes associated with vma + */ +struct xe_vma_mem_attr { + /** @preferred_loc: perferred memory_location */ + struct { + /** @preferred_loc.migration_policy: Pages migration policy */ + u32 migration_policy; + + /** + * @preferred_loc.devmem_fd: used for determining pagemap_fd + * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and + * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or + * closest device memory respectively. + */ + u32 devmem_fd; + } preferred_loc; + + /** + * @atomic_access: The atomic access type for the vma + * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE, + * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible + * values. These are defined in uapi/drm/xe_drm.h. + */ + u32 atomic_access; +}; + struct xe_vma { /** @gpuva: Base GPUVA object */ struct drm_gpuva gpuva; @@ -135,6 +162,12 @@ struct xe_vma { * Needs to be signalled before UNMAP can be processed. */ struct xe_user_fence *ufence; + + /** + * @attr: The attributes of vma which determines the migration policy + * and encoding of the PTEs for this vma. + */ + struct xe_vma_mem_attr attr; }; /** From 11974fe8c7eac3ee3f35dfe6696f097284bd67eb Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:47 +0530 Subject: [PATCH 124/169] drm/xe/vma: Move pat_index to vma attributes The PAT index determines how PTEs are encoded and can be modified by madvise. Therefore, it is now part of the vma attributes. Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-4-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_pt.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 6 +++--- drivers/gpu/drm/xe/xe_vm_types.h | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index f3a39e734a90..ba7a50bf3a2d 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -518,7 +518,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, { struct xe_pt_stage_bind_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - u16 pat_index = xe_walk->vma->pat_index; + u16 pat_index = xe_walk->vma->attr.pat_index; struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); struct xe_vm *vm = xe_walk->vm; struct xe_pt *xe_child; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 529b6767caac..bb831629d149 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1223,7 +1223,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (vm->xe->info.has_atomic_enable_pte_bit) vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; - vma->pat_index = pat_index; + vma->attr.pat_index = pat_index; if (bo) { struct drm_gpuvm_bo *vm_bo; @@ -2704,7 +2704,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, if (op->base.remap.prev) { vma = new_vma(vm, op->base.remap.prev, - old->pat_index, flags); + old->attr.pat_index, flags); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2734,7 +2734,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, if (op->base.remap.next) { vma = new_vma(vm, op->base.remap.next, - old->pat_index, flags); + old->attr.pat_index, flags); if (IS_ERR(vma)) return PTR_ERR(vma); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 8e8138c2b80a..c7b2bfa0a0d1 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -102,6 +102,11 @@ struct xe_vma_mem_attr { * values. These are defined in uapi/drm/xe_drm.h. */ u32 atomic_access; + + /** + * @pat_index: The pat index to use when encoding the PTEs for this vma. + */ + u16 pat_index; }; struct xe_vma { @@ -152,11 +157,6 @@ struct xe_vma { /** @tile_staged: bind is staged for this VMA */ u8 tile_staged; - /** - * @pat_index: The pat index to use when encoding the PTEs for this vma. - */ - u16 pat_index; - /** * @ufence: The user fence that was provided with MAP. * Needs to be signalled before UNMAP can be processed. From 29c39c56a08b238d1a069cd0e16adfbef30c4968 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:48 +0530 Subject: [PATCH 125/169] drm/xe/vma: Modify new_vma to accept struct xe_vma_mem_attr as parameter This change simplifies the logic by ensuring that remapped previous or next VMAs are created with the same memory attributes as the original VMA. By passing struct xe_vma_mem_attr as a parameter, we maintain consistency in memory attributes. -v2 *dst = *src (Matthew Brost) -v3 (Matthew Brost) Drop unnecessary helper pass attr ptr as input to new_vma and vma_create Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-5-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index bb831629d149..12b478e7ef01 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1168,7 +1168,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 start, u64 end, - u16 pat_index, unsigned int flags) + struct xe_vma_mem_attr *attr, + unsigned int flags) { struct xe_vma *vma; struct xe_tile *tile; @@ -1223,7 +1224,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (vm->xe->info.has_atomic_enable_pte_bit) vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; - vma->attr.pat_index = pat_index; + vma->attr = *attr; if (bo) { struct drm_gpuvm_bo *vm_bo; @@ -2475,7 +2476,7 @@ unwind_prefetch_ops: ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_create, ERRNO); static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, - u16 pat_index, unsigned int flags) + struct xe_vma_mem_attr *attr, unsigned int flags) { struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; struct drm_exec exec; @@ -2504,7 +2505,7 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, } vma = xe_vma_create(vm, bo, op->gem.offset, op->va.addr, op->va.addr + - op->va.range - 1, pat_index, flags); + op->va.range - 1, attr, flags); if (IS_ERR(vma)) goto err_unlock; @@ -2647,6 +2648,15 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, switch (op->base.op) { case DRM_GPUVA_OP_MAP: { + struct xe_vma_mem_attr default_attr = { + .preferred_loc = { + .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, + .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, + }, + .atomic_access = DRM_XE_ATOMIC_UNDEFINED, + .pat_index = op->map.pat_index, + }; + flags |= op->map.read_only ? VMA_CREATE_FLAG_READ_ONLY : 0; flags |= op->map.is_null ? @@ -2656,7 +2666,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, flags |= op->map.is_cpu_addr_mirror ? VMA_CREATE_FLAG_IS_SYSTEM_ALLOCATOR : 0; - vma = new_vma(vm, &op->base.map, op->map.pat_index, + vma = new_vma(vm, &op->base.map, &default_attr, flags); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2704,7 +2714,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, if (op->base.remap.prev) { vma = new_vma(vm, op->base.remap.prev, - old->attr.pat_index, flags); + &old->attr, flags); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2734,7 +2744,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, if (op->base.remap.next) { vma = new_vma(vm, op->base.remap.next, - old->attr.pat_index, flags); + &old->attr, flags); if (IS_ERR(vma)) return PTR_ERR(vma); From 186b526abd6c20713d0dc9992ad736027e866a8d Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:49 +0530 Subject: [PATCH 126/169] drm/xe/svm: Split system allocator vma incase of madvise call If the start or end of input address range lies within system allocator vma split the vma to create new vma's as per input range. v2 (Matthew Brost) - Add lockdep_assert_write for vm->lock - Remove unnecessary page aligned checks - Add kerrnel-doc and comments - Remove unnecessary unwind_ops and return v3 - Fix copying of attributes v4 - Nit fixes v5 - Squash identifier for madvise in xe_vma_ops to this patch v6/v7/v8 - Rebase on drm_gpuvm changes Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-6-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 108 +++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 2 + drivers/gpu/drm/xe/xe_vm_types.h | 1 + 3 files changed, 111 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 12b478e7ef01..9c76cb31025d 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4207,3 +4207,111 @@ void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) } kvfree(snap); } + +/** + * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops + * @vm: Pointer to the xe_vm structure + * @start: Starting input address + * @range: Size of the input range + * + * This function splits existing vma to create new vma for user provided input range + * + * Return: 0 if success + */ +int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +{ + struct drm_gpuvm_map_req map_req = { + .map.va.addr = start, + .map.va.range = range, + }; + + struct xe_vma_ops vops; + struct drm_gpuva_ops *ops = NULL; + struct drm_gpuva_op *__op; + bool is_cpu_addr_mirror = false; + bool remap_op = false; + struct xe_vma_mem_attr tmp_attr; + int err; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); + ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, &map_req); + if (IS_ERR(ops)) + return PTR_ERR(ops); + + if (list_empty(&ops->list)) { + err = 0; + goto free_ops; + } + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + + if (__op->op == DRM_GPUVA_OP_REMAP) { + xe_assert(vm->xe, !remap_op); + remap_op = true; + + if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.remap.unmap->va))) + is_cpu_addr_mirror = true; + else + is_cpu_addr_mirror = false; + } + + if (__op->op == DRM_GPUVA_OP_MAP) { + xe_assert(vm->xe, remap_op); + remap_op = false; + + /* In case of madvise ops DRM_GPUVA_OP_MAP is always after + * DRM_GPUVA_OP_REMAP, so ensure we assign op->map.is_cpu_addr_mirror true + * if REMAP is for xe_vma_is_cpu_addr_mirror vma + */ + op->map.is_cpu_addr_mirror = is_cpu_addr_mirror; + } + + print_op(vm->xe, __op); + } + + xe_vma_ops_init(&vops, vm, NULL, NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_MADVISE; + err = vm_bind_ioctl_ops_parse(vm, ops, &vops); + if (err) + goto unwind_ops; + + xe_vm_lock(vm, false); + + drm_gpuva_for_each_op(__op, ops) { + struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + struct xe_vma *vma; + + if (__op->op == DRM_GPUVA_OP_UNMAP) { + /* There should be no unmap */ + XE_WARN_ON("UNEXPECTED UNMAP"); + xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), NULL); + } else if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + /* Store attributes for REMAP UNMAPPED VMA, so they can be assigned + * to newly MAP created vma. + */ + tmp_attr = vma->attr; + xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); + } else if (__op->op == DRM_GPUVA_OP_MAP) { + vma = op->map.vma; + /* In case of madvise call, MAP will always be follwed by REMAP. + * Therefore temp_attr will always have sane values, making it safe to + * copy them to new vma. + */ + vma->attr = tmp_attr; + } + } + + xe_vm_unlock(vm); + drm_gpuva_ops_free(&vm->gpuvm, ops); + return 0; + +unwind_ops: + vm_bind_ioctl_ops_unwind(vm, &ops, 1); +free_ops: + drm_gpuva_ops_free(&vm->gpuvm, ops); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 2ecb417c19a2..ccc549e39cfc 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -171,6 +171,8 @@ static inline bool xe_vma_is_userptr(struct xe_vma *vma) struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr); +int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); + /** * to_userptr_vma() - Return a pointer to an embedding userptr vma * @vma: Pointer to the embedded struct xe_vma diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index c7b2bfa0a0d1..dde7218ceba6 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -495,6 +495,7 @@ struct xe_vma_ops { struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; /** @flag: signify the properties within xe_vma_ops*/ #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) +#define XE_VMA_OPS_FLAG_MADVISE BIT(1) u32 flags; #ifdef TEST_VM_OPS_ERROR /** @inject_error: inject error to test error handling */ From 6ad887f378c858ef3f721d5c1d8dad17842cc58b Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:50 +0530 Subject: [PATCH 127/169] drm/xe: Allow CPU address mirror VMA unbind with gpu bindings for madvise In the case of the MADVISE ioctl, if the start or end addresses fall within a VMA and existing SVM ranges are present, remove the existing SVM mappings. Then, continue with ops_parse to create new VMAs by REMAP unmapping of old one. v2 (Matthew Brost) - Use vops flag to call unmapping of ranges in vm_bind_ioctl_ops_parse - Rename the function v3 - Fix doc v4 - check if range is already in garbage collector (Matthew Brost) Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-7-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 35 +++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_svm.h | 7 +++++++ drivers/gpu/drm/xe/xe_vm.c | 8 ++++++-- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index e35c6d4def20..ce42100cb753 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -932,6 +932,41 @@ bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) return drm_gpusvm_has_mapping(&vm->svm.gpusvm, start, end); } +/** + * xe_svm_unmap_address_range - UNMAP SVM mappings and ranges + * @vm: The VM + * @start: start addr + * @end: end addr + * + * This function UNMAPS svm ranges if start or end address are inside them. + */ +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpusvm_notifier *notifier, *next; + + lockdep_assert_held_write(&vm->lock); + + drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, start, end) { + struct drm_gpusvm_range *range, *__next; + + drm_gpusvm_for_each_range_safe(range, __next, notifier, start, end) { + if (start > drm_gpusvm_range_start(range) || + end < drm_gpusvm_range_end(range)) { + if (IS_DGFX(vm->xe) && xe_svm_range_in_vram(to_xe_range(range))) + drm_gpusvm_range_evict(&vm->svm.gpusvm, range); + drm_gpusvm_range_get(range); + __xe_svm_garbage_collector(vm, to_xe_range(range)); + if (!list_empty(&to_xe_range(range)->garbage_collector_link)) { + spin_lock(&vm->svm.garbage_collector.lock); + list_del(&to_xe_range(range)->garbage_collector_link); + spin_unlock(&vm->svm.garbage_collector.lock); + } + drm_gpusvm_range_put(range); + } + } + } +} + /** * xe_svm_bo_evict() - SVM evict BO to system memory * @bo: BO to evict diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 4bdccb56d25f..184b3f4f0b2a 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -90,6 +90,8 @@ bool xe_svm_range_validate(struct xe_vm *vm, u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma); +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); + /** * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping * @range: SVM range @@ -303,6 +305,11 @@ u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vm return ULONG_MAX; } +static inline +void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) +{ +} + #define xe_svm_assert_in_notifier(...) do {} while (0) #define xe_svm_range_has_dma_mapping(...) false diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 9c76cb31025d..9fef01400c74 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2694,8 +2694,12 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, end = op->base.remap.next->va.addr; if (xe_vma_is_cpu_addr_mirror(old) && - xe_svm_has_mapping(vm, start, end)) - return -EBUSY; + xe_svm_has_mapping(vm, start, end)) { + if (vops->flags & XE_VMA_OPS_FLAG_MADVISE) + xe_svm_unmap_address_range(vm, start, end); + else + return -EBUSY; + } op->remap.start = xe_vma_start(old); op->remap.range = xe_vma_size(old); From 6ca463ef0d97a20eba559e699f0f6c4dd71e5011 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:51 +0530 Subject: [PATCH 128/169] drm/xe/svm: Add xe_svm_ranges_zap_ptes_in_range() for PTE zapping Introduce xe_svm_ranges_zap_ptes_in_range(), a function to zap page table entries (PTEs) for all SVM ranges within a user-specified address range. -v2 (Matthew Brost) Lock should be called even for tlb_invalidation v3(Matthew Brost) - Update comment - s/notifier->itree.start/drm_gpusvm_notifier_start - s/notifier->itree.last + 1/drm_gpusvm_notifier_end - use WRITE_ONCE Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-8-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_pt.c | 14 ++++++++++- drivers/gpu/drm/xe/xe_svm.c | 50 +++++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_svm.h | 8 ++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index ba7a50bf3a2d..bf50a821853e 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -950,7 +950,19 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, struct xe_pt *pt = vm->pt_root[tile->id]; u8 pt_mask = (range->tile_present & ~range->tile_invalidated); - xe_svm_assert_in_notifier(vm); + /* + * Locking rules: + * + * - notifier_lock (write): full protection against page table changes + * and MMU notifier invalidations. + * + * - notifier_lock (read) + vm_lock (write): combined protection against + * invalidations and concurrent page table modifications. (e.g., madvise) + * + */ + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || + (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + lockdep_is_held_type(&vm->lock, 0))); if (!(pt_mask & BIT(tile->id))) return false; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index ce42100cb753..c2306000f15e 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -1031,6 +1031,56 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, return err; } +/** + * xe_svm_ranges_zap_ptes_in_range - clear ptes of svm ranges in input range + * @vm: Pointer to the xe_vm structure + * @start: Start of the input range + * @end: End of the input range + * + * This function removes the page table entries (PTEs) associated + * with the svm ranges within the given input start and end + * + * Return: tile_mask for which gt's need to be tlb invalidated. + */ +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpusvm_notifier *notifier; + struct xe_svm_range *range; + u64 adj_start, adj_end; + struct xe_tile *tile; + u8 tile_mask = 0; + u8 id; + + lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && + lockdep_is_held_type(&vm->lock, 0)); + + drm_gpusvm_for_each_notifier(notifier, &vm->svm.gpusvm, start, end) { + struct drm_gpusvm_range *r = NULL; + + adj_start = max(start, drm_gpusvm_notifier_start(notifier)); + adj_end = min(end, drm_gpusvm_notifier_end(notifier)); + drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) { + range = to_xe_range(r); + for_each_tile(tile, vm->xe, id) { + if (xe_pt_zap_ptes_range(tile, vm, range)) { + tile_mask |= BIT(id); + /* + * WRITE_ONCE pairs with READ_ONCE in + * xe_vm_has_valid_gpu_mapping(). + * Must not fail after setting + * tile_invalidated and before + * TLB invalidation. + */ + WRITE_ONCE(range->tile_invalidated, + range->tile_invalidated | BIT(id)); + } + } + } + } + + return tile_mask; +} + #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile) diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 184b3f4f0b2a..046a9c4e95c2 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -92,6 +92,8 @@ u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *v void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); + /** * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping * @range: SVM range @@ -310,6 +312,12 @@ void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) { } +static inline +u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) +{ + return 0; +} + #define xe_svm_assert_in_notifier(...) do {} while (0) #define xe_svm_range_has_dma_mapping(...) false From ada7486c5668db542a7d361268df931aca5b726a Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:52 +0530 Subject: [PATCH 129/169] drm/xe: Implement madvise ioctl for xe This driver-specific ioctl enables UMDs to control the memory attributes for GPU VMAs within a specified input range. If the start or end addresses fall within an existing VMA, the VMA is split accordingly. The attributes of the VMA are modified as provided by the users. The old mappings of the VMAs are invalidated, and TLB invalidation is performed if necessary. v2(Matthew brost) - xe_vm_in_fault_mode can't be enabled by Mesa, hence allow ioctl in non fault mode too - fix tlb invalidation skip for same ranges in multiple op - use helper for tlb invalidation - use xe_svm_notifier_lock/unlock helper - s/lockdep_assert_held/lockdep_assert_held_write - Add kernel-doc v3(Matthew Brost) - make vfunc fail safe - Add sanitizing input args before vfunc v4(Matthew Brost/Shuicheng) - Make locks interruptable - Error handling fixes - vm_put fixes v5(Matthew Brost) - Flush garbage collector before any locking. - Add check for null vma Cc: Matthew Brost Cc: Shuicheng Lin Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-9-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_vm_madvise.c | 308 +++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm_madvise.h | 15 ++ 3 files changed, 324 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.c create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 85b8d3a59ef0..c52cac979e05 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -129,6 +129,7 @@ xe-y += xe_bb.o \ xe_uc.o \ xe_uc_fw.o \ xe_vm.o \ + xe_vm_madvise.o \ xe_vram.o \ xe_vram_freq.o \ xe_vsec.o \ diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c new file mode 100644 index 000000000000..b861c3349b0a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_vm_madvise.h" + +#include +#include + +#include "xe_bo.h" +#include "xe_pt.h" +#include "xe_svm.h" + +struct xe_vmas_in_madvise_range { + u64 addr; + u64 range; + struct xe_vma **vmas; + int num_vmas; + bool has_svm_vmas; + bool has_bo_vmas; + bool has_userptr_vmas; +}; + +static int get_vmas(struct xe_vm *vm, struct xe_vmas_in_madvise_range *madvise_range) +{ + u64 addr = madvise_range->addr; + u64 range = madvise_range->range; + + struct xe_vma **__vmas; + struct drm_gpuva *gpuva; + int max_vmas = 8; + + lockdep_assert_held(&vm->lock); + + madvise_range->num_vmas = 0; + madvise_range->vmas = kmalloc_array(max_vmas, sizeof(*madvise_range->vmas), GFP_KERNEL); + if (!madvise_range->vmas) + return -ENOMEM; + + vm_dbg(&vm->xe->drm, "VMA's in range: start=0x%016llx, end=0x%016llx", addr, addr + range); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, addr, addr + range) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (xe_vma_bo(vma)) + madvise_range->has_bo_vmas = true; + else if (xe_vma_is_cpu_addr_mirror(vma)) + madvise_range->has_svm_vmas = true; + else if (xe_vma_is_userptr(vma)) + madvise_range->has_userptr_vmas = true; + + if (madvise_range->num_vmas == max_vmas) { + max_vmas <<= 1; + __vmas = krealloc(madvise_range->vmas, + max_vmas * sizeof(*madvise_range->vmas), + GFP_KERNEL); + if (!__vmas) { + kfree(madvise_range->vmas); + return -ENOMEM; + } + madvise_range->vmas = __vmas; + } + + madvise_range->vmas[madvise_range->num_vmas] = vma; + (madvise_range->num_vmas)++; + } + + if (!madvise_range->num_vmas) + kfree(madvise_range->vmas); + + vm_dbg(&vm->xe->drm, "madvise_range-num_vmas = %d\n", madvise_range->num_vmas); + + return 0; +} + +static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op) +{ + /* Implementation pending */ +} + +static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op) +{ + /* Implementation pending */ +} + +static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op) +{ + /* Implementation pending */ +} + +typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + struct drm_xe_madvise *op); + +static const madvise_func madvise_funcs[] = { + [DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC] = madvise_preferred_mem_loc, + [DRM_XE_MEM_RANGE_ATTR_ATOMIC] = madvise_atomic, + [DRM_XE_MEM_RANGE_ATTR_PAT] = madvise_pat_index, +}; + +static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpuva *gpuva; + struct xe_tile *tile; + u8 id, tile_mask; + + lockdep_assert_held_write(&vm->lock); + + /* Wait for pending binds */ + if (dma_resv_wait_timeout(xe_vm_resv(vm), DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT) <= 0) + XE_WARN_ON(1); + + tile_mask = xe_svm_ranges_zap_ptes_in_range(vm, start, end); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (xe_vma_is_cpu_addr_mirror(vma) || xe_vma_is_null(vma)) + continue; + + for_each_tile(tile, vm->xe, id) { + if (xe_pt_zap_ptes(tile, vma)) { + tile_mask |= BIT(id); + + /* + * WRITE_ONCE pairs with READ_ONCE + * in xe_vm_has_valid_gpu_mapping() + */ + WRITE_ONCE(vma->tile_invalidated, + vma->tile_invalidated | BIT(id)); + } + } + } + + return tile_mask; +} + +static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end) +{ + u8 tile_mask = xe_zap_ptes_in_madvise_range(vm, start, end); + + if (!tile_mask) + return 0; + + xe_device_wmb(vm->xe); + + return xe_vm_range_tilemask_tlb_invalidation(vm, start, end, tile_mask); +} + +static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args) +{ + if (XE_IOCTL_DBG(xe, !args)) + return false; + + if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->start, SZ_4K))) + return false; + + if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->range, SZ_4K))) + return false; + + if (XE_IOCTL_DBG(xe, args->range < SZ_4K)) + return false; + + switch (args->type) { + case DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > + DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) + return false; + + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.pad)) + return false; + + if (XE_IOCTL_DBG(xe, args->atomic.reserved)) + return false; + break; + case DRM_XE_MEM_RANGE_ATTR_ATOMIC: + if (XE_IOCTL_DBG(xe, args->atomic.val > DRM_XE_ATOMIC_CPU)) + return false; + + if (XE_IOCTL_DBG(xe, args->atomic.pad)) + return false; + + if (XE_IOCTL_DBG(xe, args->atomic.reserved)) + return false; + + break; + case DRM_XE_MEM_RANGE_ATTR_PAT: + /*TODO: Add valid pat check */ + break; + default: + if (XE_IOCTL_DBG(xe, 1)) + return false; + } + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return false; + + return true; +} + +/** + * xe_vm_madvise_ioctl - Handle MADVise ioctl for a VM + * @dev: DRM device pointer + * @data: Pointer to ioctl data (drm_xe_madvise*) + * @file: DRM file pointer + * + * Handles the MADVISE ioctl to provide memory advice for vma's within + * input range. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_madvise *args = data; + struct xe_vmas_in_madvise_range madvise_range = {.addr = args->start, + .range = args->range, }; + struct xe_vm *vm; + struct drm_exec exec; + int err, attr_type; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + if (!madvise_args_are_sane(vm->xe, args)) { + err = -EINVAL; + goto put_vm; + } + + xe_svm_flush(vm); + + err = down_write_killable(&vm->lock); + if (err) + goto put_vm; + + if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { + err = -ENOENT; + goto unlock_vm; + } + + err = xe_vm_alloc_madvise_vma(vm, args->start, args->range); + if (err) + goto unlock_vm; + + err = get_vmas(vm, &madvise_range); + if (err || !madvise_range.num_vmas) + goto unlock_vm; + + if (madvise_range.has_bo_vmas) { + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT, 0); + drm_exec_until_all_locked(&exec) { + for (int i = 0; i < madvise_range.num_vmas; i++) { + struct xe_bo *bo = xe_vma_bo(madvise_range.vmas[i]); + + if (!bo) + continue; + err = drm_exec_lock_obj(&exec, &bo->ttm.base); + drm_exec_retry_on_contention(&exec); + if (err) + goto err_fini; + } + } + } + + if (madvise_range.has_userptr_vmas) { + err = down_read_interruptible(&vm->userptr.notifier_lock); + if (err) + goto err_fini; + } + + if (madvise_range.has_svm_vmas) { + err = down_read_interruptible(&vm->svm.gpusvm.notifier_lock); + if (err) + goto unlock_userptr; + } + + attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs)); + madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args); + + err = xe_vm_invalidate_madvise_range(vm, args->start, args->start + args->range); + + if (madvise_range.has_svm_vmas) + xe_svm_notifier_unlock(vm); + +unlock_userptr: + if (madvise_range.has_userptr_vmas) + up_read(&vm->userptr.notifier_lock); +err_fini: + if (madvise_range.has_bo_vmas) + drm_exec_fini(&exec); + kfree(madvise_range.vmas); + madvise_range.vmas = NULL; +unlock_vm: + up_write(&vm->lock); +put_vm: + xe_vm_put(vm); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h new file mode 100644 index 000000000000..b0e1fc445f23 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_VM_MADVISE_H_ +#define _XE_VM_MADVISE_H_ + +struct drm_device; +struct drm_file; + +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif From d6db171167f886d5f173c67044e413e14b02c880 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:53 +0530 Subject: [PATCH 130/169] drm/xe/svm: Add svm ranges migration policy on atomic access If the platform does not support atomic access on system memory, and the ranges are in system memory, but the user requires atomic accesses on the VMA, then migrate the ranges to VRAM. Apply this policy for prefetch operations as well. v2 - Drop unnecessary vm_dbg v3 (Matthew Brost) - fix atomic policy - prefetch shouldn't have any impact of atomic - bo can be accessed from vma, avoid duplicate parameter v4 (Matthew Brost) - Remove TODO comment - Fix comment - Dont allow gpu atomic ops when user is setting atomic attr as CPU v5 (Matthew Brost) - Fix atomic checks - Add userptr checks Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-10-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_pt.c | 23 ++++++++------ drivers/gpu/drm/xe/xe_svm.c | 50 ++++++++++++++++++------------ drivers/gpu/drm/xe/xe_vm.c | 39 +++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 2 ++ drivers/gpu/drm/xe/xe_vm_madvise.c | 15 ++++++++- 5 files changed, 99 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index bf50a821853e..c0a70c80dff9 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -640,28 +640,31 @@ static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { * - In all other cases device atomics will be disabled with AE=0 until an application * request differently using a ioctl like madvise. */ -static bool xe_atomic_for_vram(struct xe_vm *vm) +static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma) { + if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU) + return false; + return true; } -static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_bo *bo) +static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma) { struct xe_device *xe = vm->xe; + struct xe_bo *bo = xe_vma_bo(vma); - if (!xe->info.has_device_atomics_on_smem) + if (!xe->info.has_device_atomics_on_smem || + vma->attr.atomic_access == DRM_XE_ATOMIC_CPU) return false; + if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE) + return true; + /* * If a SMEM+LMEM allocation is backed by SMEM, a device * atomics will cause a gpu page fault and which then * gets migrated to LMEM, bind such allocations with * device atomics enabled. - * - * TODO: Revisit this. Perhaps add something like a - * fault_on_atomics_in_system UAPI flag. - * Note that this also prohibits GPU atomics in LR mode for - * userptr and system memory on DGFX. */ return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) || (bo && xe_bo_has_single_placement(bo)))); @@ -744,8 +747,8 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, goto walk_pt; if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) { - xe_walk.default_vram_pte = xe_atomic_for_vram(vm) ? XE_USM_PPGTT_PTE_AE : 0; - xe_walk.default_system_pte = xe_atomic_for_system(vm, bo) ? + xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0; + xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0; } diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index c2306000f15e..c660ccb21945 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -789,22 +789,9 @@ bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vm return true; } -/** - * xe_svm_handle_pagefault() - SVM handle page fault - * @vm: The VM. - * @vma: The CPU address mirror VMA. - * @gt: The gt upon the fault occurred. - * @fault_addr: The GPU fault address. - * @atomic: The fault atomic access bit. - * - * Create GPU bindings for a SVM page fault. Optionally migrate to device - * memory. - * - * Return: 0 on success, negative error code on error. - */ -int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, - struct xe_gt *gt, u64 fault_addr, - bool atomic) +static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool need_vram) { struct drm_gpusvm_ctx ctx = { .read_only = xe_vma_read_only(vma), @@ -812,9 +799,8 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, IS_ENABLED(CONFIG_DRM_XE_PAGEMAP), .check_pages_threshold = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) ? SZ_64K : 0, - .devmem_only = atomic && IS_DGFX(vm->xe) && - IS_ENABLED(CONFIG_DRM_XE_PAGEMAP), - .timeslice_ms = atomic && IS_DGFX(vm->xe) && + .devmem_only = need_vram && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP), + .timeslice_ms = need_vram && IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) ? vm->xe->atomic_svm_timeslice_ms : 0, }; @@ -917,6 +903,32 @@ err_out: return err; } +/** + * xe_svm_handle_pagefault() - SVM handle page fault + * @vm: The VM. + * @vma: The CPU address mirror VMA. + * @gt: The gt upon the fault occurred. + * @fault_addr: The GPU fault address. + * @atomic: The fault atomic access bit. + * + * Create GPU bindings for a SVM page fault. Optionally migrate to device + * memory. + * + * Return: 0 on success, negative error code on error. + */ +int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, + struct xe_gt *gt, u64 fault_addr, + bool atomic) +{ + int need_vram; + + need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); + if (need_vram < 0) + return need_vram; + + return __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, need_vram ? true : false); +} + /** * xe_svm_has_mapping() - SVM has mappings * @vm: The VM. diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 9fef01400c74..7bbf12742606 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4212,6 +4212,45 @@ void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) kvfree(snap); } +/** + * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations + * @xe: Pointer to the XE device structure + * @vma: Pointer to the virtual memory area (VMA) structure + * @is_atomic: In pagefault path and atomic operation + * + * This function determines whether the given VMA needs to be migrated to + * VRAM in order to do atomic GPU operation. + * + * Return: + * 1 - Migration to VRAM is required + * 0 - Migration is not required + * -EACCES - Invalid access for atomic memory attr + * + */ +int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic) +{ + if (!IS_DGFX(xe) || !is_atomic) + return 0; + + /* + * NOTE: The checks implemented here are platform-specific. For + * instance, on a device supporting CXL atomics, these would ideally + * work universally without additional handling. + */ + switch (vma->attr.atomic_access) { + case DRM_XE_ATOMIC_DEVICE: + return !xe->info.has_device_atomics_on_smem; + + case DRM_XE_ATOMIC_CPU: + return -EACCES; + + case DRM_XE_ATOMIC_UNDEFINED: + case DRM_XE_ATOMIC_GLOBAL: + default: + return 1; + } +} + /** * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops * @vm: Pointer to the xe_vm structure diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index ccc549e39cfc..dbb1bf3caf2d 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -171,6 +171,8 @@ static inline bool xe_vma_is_userptr(struct xe_vma *vma) struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr); +int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic); + int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); /** diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index b861c3349b0a..95258bb6a8ee 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -85,7 +85,20 @@ static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, struct drm_xe_madvise *op) { - /* Implementation pending */ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC); + xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU); + + for (i = 0; i < num_vmas; i++) { + if ((xe_vma_is_userptr(vmas[i]) && + !(op->atomic.val == DRM_XE_ATOMIC_DEVICE && + xe->info.has_device_atomics_on_smem))) + continue; + + vmas[i]->attr.atomic_access = op->atomic.val; + /*TODO: handle bo backed vmas */ + } } static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, From a894c27407402b6f980876603e1fa66470555ff6 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:54 +0530 Subject: [PATCH 131/169] drm/xe/madvise: Update migration policy based on preferred location When the user sets the valid devmem_fd as a preferred location, GPU fault will trigger migration to tile of device associated with devmem_fd. If the user sets an invalid devmem_fd the preferred location is current placement(smem) only. v2(Matthew Brost) - Default should be faulting tile - remove devmem_fd used as region v3 (Matthew Brost) - Add migration_policy - Fix return condition - fix migrate condition v4 -Rebase v5 - Add check for userptr and bo based vmas Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-11-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 45 +++++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_svm.h | 8 ++++++ drivers/gpu/drm/xe/xe_vm_madvise.c | 25 ++++++++++++++++- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index c660ccb21945..19585a3d9f69 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -806,6 +806,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, }; struct xe_svm_range *range; struct dma_fence *fence; + struct drm_pagemap *dpagemap; struct xe_tile *tile = gt_to_tile(gt); int migrate_try_count = ctx.devmem_only ? 3 : 1; ktime_t end = 0; @@ -835,8 +836,14 @@ retry: range_debug(range, "PAGE FAULT"); + dpagemap = xe_vma_resolve_pagemap(vma, tile); if (--migrate_try_count >= 0 && - xe_svm_range_needs_migrate_to_vram(range, vma, IS_DGFX(vm->xe))) { + xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap || ctx.devmem_only)) { + /* TODO : For multi-device dpagemap will be used to find the + * remote tile and remote device. Will need to modify + * xe_svm_alloc_vram to use dpagemap for future multi-device + * support. + */ err = xe_svm_alloc_vram(tile, range, &ctx); ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ if (err) { @@ -1100,6 +1107,37 @@ static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile) return &tile->mem.vram->dpagemap; } +/** + * xe_vma_resolve_pagemap - Resolve the appropriate DRM pagemap for a VMA + * @vma: Pointer to the xe_vma structure containing memory attributes + * @tile: Pointer to the xe_tile structure used as fallback for VRAM mapping + * + * This function determines the correct DRM pagemap to use for a given VMA. + * It first checks if a valid devmem_fd is provided in the VMA's preferred + * location. If the devmem_fd is negative, it returns NULL, indicating no + * pagemap is available and smem to be used as preferred location. + * If the devmem_fd is equal to the default faulting + * GT identifier, it returns the VRAM pagemap associated with the tile. + * + * Future support for multi-device configurations may use drm_pagemap_from_fd() + * to resolve pagemaps from arbitrary file descriptors. + * + * Return: A pointer to the resolved drm_pagemap, or NULL if none is applicable. + */ +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + s32 fd = (s32)vma->attr.preferred_loc.devmem_fd; + + if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM) + return NULL; + + if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE) + return IS_DGFX(tile_to_xe(tile)) ? tile_local_pagemap(tile) : NULL; + + /* TODO: Support multi-device with drm_pagemap_from_fd(fd) */ + return NULL; +} + /** * xe_svm_alloc_vram()- Allocate device memory pages for range, * migrating existing data. @@ -1212,6 +1250,11 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) { return 0; } + +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + return NULL; +} #endif /** diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 046a9c4e95c2..9d6a8840a8b7 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -94,6 +94,8 @@ void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end); u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile); + /** * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping * @range: SVM range @@ -318,6 +320,12 @@ u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) return 0; } +static inline +struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) +{ + return NULL; +} + #define xe_svm_assert_in_notifier(...) do {} while (0) #define xe_svm_range_has_dma_mapping(...) false diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 95258bb6a8ee..b5fc1eedf095 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -78,7 +78,23 @@ static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, struct drm_xe_madvise *op) { - /* Implementation pending */ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC); + + for (i = 0; i < num_vmas; i++) { + /*TODO: Extend attributes to bo based vmas */ + if (!xe_vma_is_cpu_addr_mirror(vmas[i])) + continue; + + vmas[i]->attr.preferred_loc.devmem_fd = op->preferred_mem_loc.devmem_fd; + + /* Till multi-device support is not added migration_policy + * is of no use and can be ignored. + */ + vmas[i]->attr.preferred_loc.migration_policy = + op->preferred_mem_loc.migration_policy; + } } static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, @@ -184,6 +200,12 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv switch (args->type) { case DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: + { + s32 fd = (s32)args->preferred_mem_loc.devmem_fd; + + if (XE_IOCTL_DBG(xe, fd < DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM)) + return false; + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) return false; @@ -194,6 +216,7 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv if (XE_IOCTL_DBG(xe, args->atomic.reserved)) return false; break; + } case DRM_XE_MEM_RANGE_ATTR_ATOMIC: if (XE_IOCTL_DBG(xe, args->atomic.val > DRM_XE_ATOMIC_CPU)) return false; From 18d36fd6d13b295ebf111de2651bb398fcd1a040 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:55 +0530 Subject: [PATCH 132/169] drm/xe/svm: Support DRM_XE_SVM_MEM_RANGE_ATTR_PAT memory attribute This attributes sets the pat_index for the svm used vma range, which is utilized to ascertain the coherence. v2 (Matthew Brost) - Pat index sanity check Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-12-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm_madvise.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index b5fc1eedf095..3bd52063f9c2 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -9,6 +9,7 @@ #include #include "xe_bo.h" +#include "xe_pat.h" #include "xe_pt.h" #include "xe_svm.h" @@ -121,7 +122,12 @@ static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, struct drm_xe_madvise *op) { - /* Implementation pending */ + int i; + + xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PAT); + + for (i = 0; i < num_vmas; i++) + vmas[i]->attr.pat_index = op->pat_index.val; } typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, @@ -229,8 +235,22 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv break; case DRM_XE_MEM_RANGE_ATTR_PAT: - /*TODO: Add valid pat check */ + { + u16 coh_mode = xe_pat_index_get_coh_mode(xe, args->pat_index.val); + + if (XE_IOCTL_DBG(xe, !coh_mode)) + return false; + + if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) + return false; + + if (XE_IOCTL_DBG(xe, args->pat_index.pad)) + return false; + + if (XE_IOCTL_DBG(xe, args->pat_index.reserved)) + return false; break; + } default: if (XE_IOCTL_DBG(xe, 1)) return false; From fa1a82c985dba642de66f0a1918fc531007bf90f Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:56 +0530 Subject: [PATCH 133/169] drm/xe/uapi: Add flag for consulting madvise hints on svm prefetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce flag DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC to ensure prefetching in madvise-advised memory regions v2 (Matthew Brost) - Add kernel-doc v3 (Matthew Brost) - Fix kernel-doc Cc: Matthew Brost Reviewed-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://lore.kernel.org/r/20250821173104.3030148-13-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- include/uapi/drm/xe_drm.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4e6e9a9164ee..115b9bca2a25 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1010,6 +1010,10 @@ struct drm_xe_vm_destroy { * valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address * mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO * handle MBZ, and the BO offset MBZ. + * + * The @prefetch_mem_region_instance for %DRM_XE_VM_BIND_OP_PREFETCH can also be: + * - %DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC, which ensures prefetching occurs in + * the memory region advised by madvise. */ struct drm_xe_vm_bind_op { /** @extensions: Pointer to the first extension struct, if any */ @@ -1115,6 +1119,7 @@ struct drm_xe_vm_bind_op { /** @flags: Bind flags */ __u32 flags; +#define DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC -1 /** * @prefetch_mem_region_instance: Memory region to prefetch VMA to. * It is a region instance, not a mask. From c1bb69a2e8e2d55c3725e304d8b8fd189ee43fbe Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:57 +0530 Subject: [PATCH 134/169] drm/xe/svm: Consult madvise preferred location in prefetch When prefetch region is DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC, prefetch svm ranges to preferred location provided by madvise. v2 (Matthew Brost) - Fix region, devmem_fd usages - consult madvise is applicable for other vma's too. v3 - Fix atomic handling v4 - Fix xe_svm_range_validate to check for DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC too. Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-14-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 47 ++++++++++++++++++++------------ drivers/gpu/drm/xe/xe_vm_types.h | 7 +++-- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 7bbf12742606..c765c96b9bde 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -38,6 +38,7 @@ #include "xe_res_cursor.h" #include "xe_svm.h" #include "xe_sync.h" +#include "xe_tile.h" #include "xe_trace_bo.h" #include "xe_wa.h" #include "xe_hmm.h" @@ -2400,9 +2401,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, __xe_vm_needs_clear_scratch_pages(vm, flags); } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + struct xe_tile *tile; struct xe_svm_range *svm_range; struct drm_gpusvm_ctx ctx = {}; - struct xe_tile *tile; + struct drm_pagemap *dpagemap; u8 id, tile_mask = 0; u32 i; @@ -2419,8 +2421,24 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, tile_mask |= 0x1 << id; xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); - op->prefetch_range.region = prefetch_region; op->prefetch_range.ranges_count = 0; + tile = NULL; + + if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { + dpagemap = xe_vma_resolve_pagemap(vma, + xe_device_get_root_tile(vm->xe)); + /* + * TODO: Once multigpu support is enabled will need + * something to dereference tile from dpagemap. + */ + if (dpagemap) + tile = xe_device_get_root_tile(vm->xe); + } else if (prefetch_region) { + tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - + XE_PL_VRAM0]; + } + + op->prefetch_range.tile = tile; alloc_next_range: svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx); @@ -2439,7 +2457,7 @@ alloc_next_range: goto unwind_prefetch_ops; } - if (xe_svm_range_validate(vm, svm_range, tile_mask, !!prefetch_region)) { + if (xe_svm_range_validate(vm, svm_range, tile_mask, !!tile)) { xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); goto check_next_range; } @@ -2935,30 +2953,26 @@ static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op) { bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + struct xe_tile *tile = op->prefetch_range.tile; int err = 0; struct xe_svm_range *svm_range; struct drm_gpusvm_ctx ctx = {}; - struct xe_tile *tile; unsigned long i; - u32 region; if (!xe_vma_is_cpu_addr_mirror(vma)) return 0; - region = op->prefetch_range.region; - ctx.read_only = xe_vma_read_only(vma); ctx.devmem_possible = devmem_possible; ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0; /* TODO: Threading the migration */ xa_for_each(&op->prefetch_range.range, i, svm_range) { - if (!region) + if (!tile) xe_svm_range_migrate_to_smem(vm, svm_range); - if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, region)) { - tile = &vm->xe->tiles[region_to_mem_type[region] - XE_PL_VRAM0]; + if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, !!tile)) { err = xe_svm_alloc_vram(tile, svm_range, &ctx); if (err) { drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n", @@ -3021,12 +3035,11 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); u32 region; - if (xe_vma_is_cpu_addr_mirror(vma)) - region = op->prefetch_range.region; - else + if (!xe_vma_is_cpu_addr_mirror(vma)) { region = op->prefetch.region; - - xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type)); + xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || + region <= ARRAY_SIZE(region_to_mem_type)); + } err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.prefetch.va), @@ -3444,8 +3457,8 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, op == DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, prefetch_region && op != DRM_XE_VM_BIND_OP_PREFETCH) || - XE_IOCTL_DBG(xe, !(BIT(prefetch_region) & - xe->info.mem_region_mask)) || + XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && + !(BIT(prefetch_region) & xe->info.mem_region_mask))) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_UNMAP)) { err = -EINVAL; diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index dde7218ceba6..5704f379c225 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -428,8 +428,11 @@ struct xe_vma_op_prefetch_range { struct xarray range; /** @ranges_count: number of svm ranges to map */ u32 ranges_count; - /** @region: memory region to prefetch to */ - u32 region; + /** + * @tile: Pointer to the tile structure containing memory to prefetch. + * NULL if prefetch requested region is smem + */ + struct xe_tile *tile; }; /** enum xe_vma_op_flags - flags for VMA operation */ From 072e299982635aa8d96683071930e3bf0a1c1a2f Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:58 +0530 Subject: [PATCH 135/169] drm/xe/bo: Add attributes field to xe_bo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single BO can be linked to multiple VMAs, making VMA attributes insufficient for determining the placement and PTE update attributes of the BO. To address this, an attributes field has been added to the BO. Reviewed-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://lore.kernel.org/r/20250821173104.3030148-15-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_bo_types.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index cf604adc13a3..314652afdca7 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -61,6 +61,14 @@ struct xe_bo { */ struct list_head client_link; #endif + /** @attr: User controlled attributes for bo */ + struct { + /** + * @atomic_access: type of atomic access bo needs + * protected by bo dma-resv lock + */ + u32 atomic_access; + } attr; /** * @pxp_key_instance: PXP key instance this BO was created against. A * 0 in this variable indicates that the BO does not use PXP encryption. From 293032eec4baa04374d62dd44de61e355296ad32 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:00:59 +0530 Subject: [PATCH 136/169] drm/xe/bo: Update atomic_access attribute on madvise Update the bo_atomic_access based on user-provided input and determine the migration to smem during a CPU fault v2 (Matthew Brost) - Avoid cpu unmapping if bo is already in smem - check atomics on smem too for ioctl - Add comments v3 - Avoid migration in prefetch v4 (Matthew Brost) - make sanity check function bool - add assert for smem placement - fix doc v5 (Matthew Brost) - NACK atomic fault with DRM_XE_ATOMIC_CPU Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-16-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_bo.c | 29 ++++++++++++-- drivers/gpu/drm/xe/xe_gt_pagefault.c | 35 ++++++----------- drivers/gpu/drm/xe/xe_vm.c | 7 +++- drivers/gpu/drm/xe/xe_vm_madvise.c | 59 +++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 0a6a32cfb771..7d1ff642b02a 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1712,6 +1712,18 @@ static void xe_gem_object_close(struct drm_gem_object *obj, } } +static bool should_migrate_to_smem(struct xe_bo *bo) +{ + /* + * NOTE: The following atomic checks are platform-specific. For example, + * if a device supports CXL atomics, these may not be necessary or + * may behave differently. + */ + + return bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL || + bo->attr.atomic_access == DRM_XE_ATOMIC_CPU; +} + static vm_fault_t xe_gem_fault(struct vm_fault *vmf) { struct ttm_buffer_object *tbo = vmf->vma->vm_private_data; @@ -1720,7 +1732,7 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) struct xe_bo *bo = ttm_to_xe_bo(tbo); bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK; vm_fault_t ret; - int idx; + int idx, r = 0; if (needs_rpm) xe_pm_runtime_get(xe); @@ -1732,8 +1744,19 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) if (drm_dev_enter(ddev, &idx)) { trace_xe_bo_cpu_fault(bo); - ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, - TTM_BO_VM_NUM_PREFAULT); + if (should_migrate_to_smem(bo)) { + xe_assert(xe, bo->flags & XE_BO_FLAG_SYSTEM); + + r = xe_bo_migrate(bo, XE_PL_TT); + if (r == -EBUSY || r == -ERESTARTSYS || r == -EINTR) + ret = VM_FAULT_NOPAGE; + else if (r) + ret = VM_FAULT_SIGBUS; + } + if (!ret) + ret = ttm_bo_vm_fault_reserved(vmf, + vmf->vma->vm_page_prot, + TTM_BO_VM_NUM_PREFAULT); drm_dev_exit(idx); if (ret == VM_FAULT_RETRY && diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index ab43dec52776..4ea30fbce9bd 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -75,7 +75,7 @@ static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) } static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, - bool atomic, struct xe_vram_region *vram) + bool need_vram_move, struct xe_vram_region *vram) { struct xe_bo *bo = xe_vma_bo(vma); struct xe_vm *vm = xe_vma_vm(vma); @@ -85,26 +85,13 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, if (err) return err; - if (atomic && vram) { - xe_assert(vm->xe, IS_DGFX(vm->xe)); + if (!bo) + return 0; - if (xe_vma_is_userptr(vma)) { - err = -EACCES; - return err; - } + err = need_vram_move ? xe_bo_migrate(bo, vram->placement) : + xe_bo_validate(bo, vm, true); - /* Migrate to VRAM, move should invalidate the VMA first */ - err = xe_bo_migrate(bo, vram->placement); - if (err) - return err; - } else if (bo) { - /* Create backing store if needed */ - err = xe_bo_validate(bo, vm, true); - if (err) - return err; - } - - return 0; + return err; } static int handle_vma_pagefault(struct xe_gt *gt, struct xe_vma *vma, @@ -115,10 +102,14 @@ static int handle_vma_pagefault(struct xe_gt *gt, struct xe_vma *vma, struct drm_exec exec; struct dma_fence *fence; ktime_t end = 0; - int err; + int err, needs_vram; lockdep_assert_held_write(&vm->lock); + needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); + if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma))) + return needs_vram < 0 ? needs_vram : -EACCES; + xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1); xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB, xe_vma_size(vma) / 1024); @@ -141,7 +132,7 @@ retry_userptr: /* Lock VM and BOs dma-resv */ drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { - err = xe_pf_begin(&exec, vma, atomic, tile->mem.vram); + err = xe_pf_begin(&exec, vma, needs_vram == 1, tile->mem.vram); drm_exec_retry_on_contention(&exec); if (xe_vm_validate_should_retry(&exec, err, &end)) err = -EAGAIN; @@ -576,7 +567,7 @@ static int handle_acc(struct xe_gt *gt, struct acc *acc) /* Lock VM and BOs dma-resv */ drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { - ret = xe_pf_begin(&exec, vma, true, tile->mem.vram); + ret = xe_pf_begin(&exec, vma, IS_DGFX(vm->xe), tile->mem.vram); drm_exec_retry_on_contention(&exec); if (ret) break; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c765c96b9bde..6574ae56a730 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4242,15 +4242,18 @@ void xe_vm_snapshot_free(struct xe_vm_snapshot *snap) */ int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic) { + u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access : + vma->attr.atomic_access; + if (!IS_DGFX(xe) || !is_atomic) - return 0; + return false; /* * NOTE: The checks implemented here are platform-specific. For * instance, on a device supporting CXL atomics, these would ideally * work universally without additional handling. */ - switch (vma->attr.atomic_access) { + switch (atomic_access) { case DRM_XE_ATOMIC_DEVICE: return !xe->info.has_device_atomics_on_smem; diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 3bd52063f9c2..212a03178d65 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -102,6 +102,7 @@ static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, struct drm_xe_madvise *op) { + struct xe_bo *bo; int i; xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC); @@ -114,7 +115,19 @@ static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, continue; vmas[i]->attr.atomic_access = op->atomic.val; - /*TODO: handle bo backed vmas */ + + bo = xe_vma_bo(vmas[i]); + if (!bo) + continue; + + xe_bo_assert_held(bo); + bo->attr.atomic_access = op->atomic.val; + + /* Invalidate cpu page table, so bo can migrate to smem in next access */ + if (xe_bo_is_vram(bo) && + (bo->attr.atomic_access == DRM_XE_ATOMIC_CPU || + bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL)) + ttm_bo_unmap_virtual(&bo->ttm); } } @@ -262,6 +275,41 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv return true; } +static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, + int num_vmas, u32 atomic_val) +{ + struct xe_device *xe = vm->xe; + struct xe_bo *bo; + int i; + + for (i = 0; i < num_vmas; i++) { + bo = xe_vma_bo(vmas[i]); + if (!bo) + continue; + /* + * NOTE: The following atomic checks are platform-specific. For example, + * if a device supports CXL atomics, these may not be necessary or + * may behave differently. + */ + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_CPU && + !(bo->flags & XE_BO_FLAG_SYSTEM))) + return false; + + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_DEVICE && + !(bo->flags & XE_BO_FLAG_VRAM0) && + !(bo->flags & XE_BO_FLAG_VRAM1) && + !(bo->flags & XE_BO_FLAG_SYSTEM && + xe->info.has_device_atomics_on_smem))) + return false; + + if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_GLOBAL && + (!(bo->flags & XE_BO_FLAG_SYSTEM) || + (!(bo->flags & XE_BO_FLAG_VRAM0) && + !(bo->flags & XE_BO_FLAG_VRAM1))))) + return false; + } + return true; +} /** * xe_vm_madvise_ioctl - Handle MADVise ioctl for a VM * @dev: DRM device pointer @@ -313,6 +361,15 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil goto unlock_vm; if (madvise_range.has_bo_vmas) { + if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { + if (!check_bo_args_are_sane(vm, madvise_range.vmas, + madvise_range.num_vmas, + args->atomic.val)) { + err = -EINVAL; + goto unlock_vm; + } + } + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT, 0); drm_exec_until_all_locked(&exec) { for (int i = 0; i < madvise_range.num_vmas; i++) { From 002f817d61ce2cfb0be850c3a6036adfbb4fd5e9 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:01:00 +0530 Subject: [PATCH 137/169] drm/xe/madvise: Skip vma invalidation if mem attr are unchanged If a VMA within the madvise input range already has the same memory attribute as the one requested by the user, skip PTE zapping for that VMA to avoid unnecessary invalidation. v2 (Matthew Brost) - fix skip_invalidation for new attributes - s/u32/bool - Remove unnecessary assignment for kzalloc'ed Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-17-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm_madvise.c | 68 ++++++++++++++++++++---------- drivers/gpu/drm/xe/xe_vm_types.h | 6 +++ 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 212a03178d65..7813bdedacaa 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -85,16 +85,20 @@ static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, for (i = 0; i < num_vmas; i++) { /*TODO: Extend attributes to bo based vmas */ - if (!xe_vma_is_cpu_addr_mirror(vmas[i])) - continue; - - vmas[i]->attr.preferred_loc.devmem_fd = op->preferred_mem_loc.devmem_fd; - - /* Till multi-device support is not added migration_policy - * is of no use and can be ignored. - */ - vmas[i]->attr.preferred_loc.migration_policy = + if ((vmas[i]->attr.preferred_loc.devmem_fd == op->preferred_mem_loc.devmem_fd && + vmas[i]->attr.preferred_loc.migration_policy == + op->preferred_mem_loc.migration_policy) || + !xe_vma_is_cpu_addr_mirror(vmas[i])) { + vmas[i]->skip_invalidation = true; + } else { + vmas[i]->skip_invalidation = false; + vmas[i]->attr.preferred_loc.devmem_fd = op->preferred_mem_loc.devmem_fd; + /* Till multi-device support is not added migration_policy + * is of no use and can be ignored. + */ + vmas[i]->attr.preferred_loc.migration_policy = op->preferred_mem_loc.migration_policy; + } } } @@ -109,17 +113,27 @@ static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU); for (i = 0; i < num_vmas; i++) { - if ((xe_vma_is_userptr(vmas[i]) && - !(op->atomic.val == DRM_XE_ATOMIC_DEVICE && - xe->info.has_device_atomics_on_smem))) + if (xe_vma_is_userptr(vmas[i]) && + !(op->atomic.val == DRM_XE_ATOMIC_DEVICE && + xe->info.has_device_atomics_on_smem)) { + vmas[i]->skip_invalidation = true; continue; + } + + if (vmas[i]->attr.atomic_access == op->atomic.val) { + vmas[i]->skip_invalidation = true; + } else { + vmas[i]->skip_invalidation = false; + vmas[i]->attr.atomic_access = op->atomic.val; + } vmas[i]->attr.atomic_access = op->atomic.val; bo = xe_vma_bo(vmas[i]); - if (!bo) + if (!bo || bo->attr.atomic_access == op->atomic.val) continue; + vmas[i]->skip_invalidation = false; xe_bo_assert_held(bo); bo->attr.atomic_access = op->atomic.val; @@ -139,8 +153,14 @@ static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PAT); - for (i = 0; i < num_vmas; i++) - vmas[i]->attr.pat_index = op->pat_index.val; + for (i = 0; i < num_vmas; i++) { + if (vmas[i]->attr.pat_index == op->pat_index.val) { + vmas[i]->skip_invalidation = true; + } else { + vmas[i]->skip_invalidation = false; + vmas[i]->attr.pat_index = op->pat_index.val; + } + } } typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, @@ -157,7 +177,7 @@ static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) { struct drm_gpuva *gpuva; struct xe_tile *tile; - u8 id, tile_mask; + u8 id, tile_mask = 0; lockdep_assert_held_write(&vm->lock); @@ -166,17 +186,20 @@ static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) false, MAX_SCHEDULE_TIMEOUT) <= 0) XE_WARN_ON(1); - tile_mask = xe_svm_ranges_zap_ptes_in_range(vm, start, end); - drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { struct xe_vma *vma = gpuva_to_vma(gpuva); - if (xe_vma_is_cpu_addr_mirror(vma) || xe_vma_is_null(vma)) + if (vma->skip_invalidation || xe_vma_is_null(vma)) continue; - for_each_tile(tile, vm->xe, id) { - if (xe_pt_zap_ptes(tile, vma)) { - tile_mask |= BIT(id); + if (xe_vma_is_cpu_addr_mirror(vma)) { + tile_mask |= xe_svm_ranges_zap_ptes_in_range(vm, + xe_vma_start(vma), + xe_vma_end(vma)); + } else { + for_each_tile(tile, vm->xe, id) { + if (xe_pt_zap_ptes(tile, vma)) { + tile_mask |= BIT(id); /* * WRITE_ONCE pairs with READ_ONCE @@ -184,6 +207,7 @@ static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) */ WRITE_ONCE(vma->tile_invalidated, vma->tile_invalidated | BIT(id)); + } } } } diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 5704f379c225..35b6344158f5 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -157,6 +157,12 @@ struct xe_vma { /** @tile_staged: bind is staged for this VMA */ u8 tile_staged; + /** + * @skip_invalidation: Used in madvise to avoid invalidation + * if mem attributes doesn't change + */ + bool skip_invalidation; + /** * @ufence: The user fence that was provided with MAP. * Needs to be signalled before UNMAP can be processed. From 58dc430d89ae2cf41a8919ac99c36c4a17a3fdc3 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:01:01 +0530 Subject: [PATCH 138/169] drm/xe/vm: Add helper to check for default VMA memory attributes Introduce a new helper function `xe_vma_has_default_mem_attrs()` to determine whether a VMA's memory attributes are set to their default values. This includes checks for atomic access, PAT index, and preferred location. Also, add a new field `default_pat_index` to `struct xe_vma_mem_attr` to track the initial PAT index set during the first bind. This helps distinguish between default and user-modified pat index, such as those changed via madvise. Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-18-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 2 ++ drivers/gpu/drm/xe/xe_vm_types.h | 6 ++++++ 3 files changed, 32 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 6574ae56a730..e9615d5ef944 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2640,6 +2640,29 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) return err; } +/** + * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes + * @vma: Pointer to the xe_vma structure to check + * + * This function determines whether the given VMA (Virtual Memory Area) + * has its memory attributes set to their default values. Specifically, + * it checks the following conditions: + * + * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED` + * - `pat_index` is equal to `default_pat_index` + * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE` + * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES` + * + * Return: true if all attributes are at their default values, false otherwise. + */ +bool xe_vma_has_default_mem_attrs(struct xe_vma *vma) +{ + return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED && + vma->attr.pat_index == vma->attr.default_pat_index && + vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && + vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES); +} + static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, struct xe_vma_ops *vops) { @@ -2672,6 +2695,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, }, .atomic_access = DRM_XE_ATOMIC_UNDEFINED, + .default_pat_index = op->map.pat_index, .pat_index = op->map.pat_index, }; diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index dbb1bf3caf2d..1a4d67f29c6b 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -66,6 +66,8 @@ static inline bool xe_vm_is_closed_or_banned(struct xe_vm *vm) struct xe_vma * xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range); +bool xe_vma_has_default_mem_attrs(struct xe_vma *vma); + /** * xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs * @vm: The vm diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 35b6344158f5..b5108d010786 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -103,8 +103,14 @@ struct xe_vma_mem_attr { */ u32 atomic_access; + /** + * @default_pat_index: The pat index for VMA set during first bind by user. + */ + u16 default_pat_index; + /** * @pat_index: The pat index to use when encoding the PTEs for this vma. + * same as default_pat_index unless overwritten by madvise. */ u16 pat_index; }; From a2eb8aec3ebe474ce0fe0b6ebb18aade8c1a5c00 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:01:02 +0530 Subject: [PATCH 139/169] drm/xe: Reset VMA attributes to default in SVM garbage collector Restore default memory attributes for VMAs during garbage collection if they were modified by madvise. Reuse existing VMA if fully overlapping; otherwise, allocate a new mirror VMA. v2 (Matthew Brost) - Add helper for vma split - Add retry to get updated vma v3 - Rebase on gpuvm layer Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-19-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 80 ++++++++++++++++-- drivers/gpu/drm/xe/xe_vm.c | 160 ++++++++++++++++++++++++++---------- drivers/gpu/drm/xe/xe_vm.h | 2 + 3 files changed, 192 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 19585a3d9f69..0596039ef0a1 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -253,10 +253,56 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm, return 0; } +static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 range_end) +{ + struct xe_vma *vma; + struct xe_vma_mem_attr default_attr = { + .preferred_loc = { + .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, + .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, + }, + .atomic_access = DRM_XE_ATOMIC_UNDEFINED, + }; + int err = 0; + + vma = xe_vm_find_vma_by_addr(vm, range_start); + if (!vma) + return -EINVAL; + + if (xe_vma_has_default_mem_attrs(vma)) + return 0; + + vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx", + xe_vma_start(vma), xe_vma_end(vma)); + + if (xe_vma_start(vma) == range_start && xe_vma_end(vma) == range_end) { + default_attr.pat_index = vma->attr.default_pat_index; + default_attr.default_pat_index = vma->attr.default_pat_index; + vma->attr = default_attr; + } else { + vm_dbg(&vm->xe->drm, "Split VMA start=0x%016llx, vma_end=0x%016llx", + range_start, range_end); + err = xe_vm_alloc_cpu_addr_mirror_vma(vm, range_start, range_end - range_start); + if (err) { + drm_warn(&vm->xe->drm, "VMA SPLIT failed: %pe\n", ERR_PTR(err)); + xe_vm_kill(vm, true); + return err; + } + } + + /* + * On call from xe_svm_handle_pagefault original VMA might be changed + * signal this to lookup for VMA again. + */ + return -EAGAIN; +} + static int xe_svm_garbage_collector(struct xe_vm *vm) { struct xe_svm_range *range; - int err; + u64 range_start; + u64 range_end; + int err, ret = 0; lockdep_assert_held_write(&vm->lock); @@ -271,6 +317,9 @@ static int xe_svm_garbage_collector(struct xe_vm *vm) if (!range) break; + range_start = xe_svm_range_start(range); + range_end = xe_svm_range_end(range); + list_del(&range->garbage_collector_link); spin_unlock(&vm->svm.garbage_collector.lock); @@ -283,11 +332,19 @@ static int xe_svm_garbage_collector(struct xe_vm *vm) return err; } + err = xe_svm_range_set_default_attr(vm, range_start, range_end); + if (err) { + if (err == -EAGAIN) + ret = -EAGAIN; + else + return err; + } + spin_lock(&vm->svm.garbage_collector.lock); } spin_unlock(&vm->svm.garbage_collector.lock); - return 0; + return ret; } static void xe_svm_garbage_collector_work_func(struct work_struct *w) @@ -927,13 +984,26 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, struct xe_gt *gt, u64 fault_addr, bool atomic) { - int need_vram; - + int need_vram, ret; +retry: need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); if (need_vram < 0) return need_vram; - return __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, need_vram ? true : false); + ret = __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, + need_vram ? true : false); + if (ret == -EAGAIN) { + /* + * Retry once on -EAGAIN to re-lookup the VMA, as the original VMA + * may have been split by xe_svm_range_set_default_attr. + */ + vma = xe_vm_find_vma_by_addr(vm, fault_addr); + if (!vma) + return -EINVAL; + + goto retry; + } + return ret; } /** diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index e9615d5ef944..ae95407691d8 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4291,35 +4291,26 @@ int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool i } } -/** - * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops - * @vm: Pointer to the xe_vm structure - * @start: Starting input address - * @range: Size of the input range - * - * This function splits existing vma to create new vma for user provided input range - * - * Return: 0 if success - */ -int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +static int xe_vm_alloc_vma(struct xe_vm *vm, + struct drm_gpuvm_map_req *map_req, + bool is_madvise) { - struct drm_gpuvm_map_req map_req = { - .map.va.addr = start, - .map.va.range = range, - }; - struct xe_vma_ops vops; struct drm_gpuva_ops *ops = NULL; struct drm_gpuva_op *__op; bool is_cpu_addr_mirror = false; bool remap_op = false; struct xe_vma_mem_attr tmp_attr; + u16 default_pat; int err; lockdep_assert_held_write(&vm->lock); - vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); - ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, &map_req); + if (is_madvise) + ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req); + else + ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req); + if (IS_ERR(ops)) return PTR_ERR(ops); @@ -4330,33 +4321,57 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) drm_gpuva_for_each_op(__op, ops) { struct xe_vma_op *op = gpuva_op_to_vma_op(__op); + struct xe_vma *vma = NULL; - if (__op->op == DRM_GPUVA_OP_REMAP) { - xe_assert(vm->xe, !remap_op); - remap_op = true; + if (!is_madvise) { + if (__op->op == DRM_GPUVA_OP_UNMAP) { + vma = gpuva_to_vma(op->base.unmap.va); + XE_WARN_ON(!xe_vma_has_default_mem_attrs(vma)); + default_pat = vma->attr.default_pat_index; + } - if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.remap.unmap->va))) - is_cpu_addr_mirror = true; - else - is_cpu_addr_mirror = false; + if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + default_pat = vma->attr.default_pat_index; + } + + if (__op->op == DRM_GPUVA_OP_MAP) { + op->map.is_cpu_addr_mirror = true; + op->map.pat_index = default_pat; + } + } else { + if (__op->op == DRM_GPUVA_OP_REMAP) { + vma = gpuva_to_vma(op->base.remap.unmap->va); + xe_assert(vm->xe, !remap_op); + xe_assert(vm->xe, xe_vma_has_no_bo(vma)); + remap_op = true; + + if (xe_vma_is_cpu_addr_mirror(vma)) + is_cpu_addr_mirror = true; + else + is_cpu_addr_mirror = false; + } + + if (__op->op == DRM_GPUVA_OP_MAP) { + xe_assert(vm->xe, remap_op); + remap_op = false; + /* + * In case of madvise ops DRM_GPUVA_OP_MAP is + * always after DRM_GPUVA_OP_REMAP, so ensure + * we assign op->map.is_cpu_addr_mirror true + * if REMAP is for xe_vma_is_cpu_addr_mirror vma + */ + op->map.is_cpu_addr_mirror = is_cpu_addr_mirror; + } } - - if (__op->op == DRM_GPUVA_OP_MAP) { - xe_assert(vm->xe, remap_op); - remap_op = false; - - /* In case of madvise ops DRM_GPUVA_OP_MAP is always after - * DRM_GPUVA_OP_REMAP, so ensure we assign op->map.is_cpu_addr_mirror true - * if REMAP is for xe_vma_is_cpu_addr_mirror vma - */ - op->map.is_cpu_addr_mirror = is_cpu_addr_mirror; - } - print_op(vm->xe, __op); } xe_vma_ops_init(&vops, vm, NULL, NULL, 0); - vops.flags |= XE_VMA_OPS_FLAG_MADVISE; + + if (is_madvise) + vops.flags |= XE_VMA_OPS_FLAG_MADVISE; + err = vm_bind_ioctl_ops_parse(vm, ops, &vops); if (err) goto unwind_ops; @@ -4368,15 +4383,20 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) struct xe_vma *vma; if (__op->op == DRM_GPUVA_OP_UNMAP) { - /* There should be no unmap */ - XE_WARN_ON("UNEXPECTED UNMAP"); - xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), NULL); + vma = gpuva_to_vma(op->base.unmap.va); + /* There should be no unmap for madvise */ + if (is_madvise) + XE_WARN_ON("UNEXPECTED UNMAP"); + + xe_vma_destroy(vma, NULL); } else if (__op->op == DRM_GPUVA_OP_REMAP) { vma = gpuva_to_vma(op->base.remap.unmap->va); - /* Store attributes for REMAP UNMAPPED VMA, so they can be assigned - * to newly MAP created vma. + /* In case of madvise ops Store attributes for REMAP UNMAPPED + * VMA, so they can be assigned to newly MAP created vma. */ - tmp_attr = vma->attr; + if (is_madvise) + tmp_attr = vma->attr; + xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); } else if (__op->op == DRM_GPUVA_OP_MAP) { vma = op->map.vma; @@ -4384,7 +4404,8 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) * Therefore temp_attr will always have sane values, making it safe to * copy them to new vma. */ - vma->attr = tmp_attr; + if (is_madvise) + vma->attr = tmp_attr; } } @@ -4398,3 +4419,52 @@ free_ops: drm_gpuva_ops_free(&vm->gpuvm, ops); return err; } + +/** + * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops + * @vm: Pointer to the xe_vm structure + * @start: Starting input address + * @range: Size of the input range + * + * This function splits existing vma to create new vma for user provided input range + * + * Return: 0 if success + */ +int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +{ + struct drm_gpuvm_map_req map_req = { + .map.va.addr = start, + .map.va.range = range, + }; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); + + return xe_vm_alloc_vma(vm, &map_req, true); +} + +/** + * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma + * @vm: Pointer to the xe_vm structure + * @start: Starting input address + * @range: Size of the input range + * + * This function splits/merges existing vma to create new vma for user provided input range + * + * Return: 0 if success + */ +int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range) +{ + struct drm_gpuvm_map_req map_req = { + .map.va.addr = start, + .map.va.range = range, + }; + + lockdep_assert_held_write(&vm->lock); + + vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", + start, range); + + return xe_vm_alloc_vma(vm, &map_req, false); +} diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 1a4d67f29c6b..015ddb8b8f99 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -177,6 +177,8 @@ int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool i int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); +int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t addr, uint64_t size); + /** * to_userptr_vma() - Return a pointer to an embedding userptr vma * @vma: Pointer to the embedded struct xe_vma From e80b05b09f0e38aabcd5185dc292db0eff7e6e22 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:01:03 +0530 Subject: [PATCH 140/169] drm/xe: Enable madvise ioctl for xe Ioctl enables setting up of memory attributes in user provided range. Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-20-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_device.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index d4e186c6545a..42eb55fd80ce 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -64,6 +64,7 @@ #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_sys_mgr.h" #include "xe_vm.h" +#include "xe_vm_madvise.h" #include "xe_vram.h" #include "xe_vram_types.h" #include "xe_vsec.h" @@ -202,6 +203,7 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), }; static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) From 418807860e94eb9c2fe07a6f5bf67de4c59a97e4 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 21 Aug 2025 23:01:04 +0530 Subject: [PATCH 141/169] drm/xe/uapi: Add UAPI for querying VMA count and memory attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the DRM_IOCTL_XE_VM_QUERY_MEMORY_RANGE_ATTRS ioctl to allow userspace to query memory attributes of VMAs within a user specified virtual address range. Userspace first calls the ioctl with num_mem_ranges = 0, sizeof_mem_ranges_attr = 0 and vector_of_vma_mem_attr = NULL to retrieve the number of memory ranges (vmas) and size of each memory range attribute. Then, it allocates a buffer of that size and calls the ioctl again to fill the buffer with memory range attributes. This two-step interface allows userspace to first query the required buffer size, then retrieve detailed attributes efficiently. v2 (Matthew Brost) - Use same ioctl to overload functionality v3 - Add kernel-doc v4 - Make uapi future proof by passing struct size (Matthew Brost) - make lock interruptible (Matthew Brost) - set reserved bits to zero (Matthew Brost) - s/__copy_to_user/copy_to_user (Matthew Brost) - Avod using VMA term in uapi (Thomas) - xe_vm_put(vm) is missing (Shuicheng) v5 - Nits - Fix kernel-doc Cc: Matthew Brost Cc: Shuicheng Lin Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250821173104.3030148-21-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_device.c | 2 + drivers/gpu/drm/xe/xe_vm.c | 102 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 2 +- include/uapi/drm/xe_drm.h | 140 +++++++++++++++++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 42eb55fd80ce..90617e627bc4 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -204,6 +204,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, + DRM_RENDER_ALLOW), }; static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index ae95407691d8..e913efa2057b 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2196,6 +2196,108 @@ int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, return err; } +static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end) +{ + struct drm_gpuva *gpuva; + u32 num_vmas = 0; + + lockdep_assert_held(&vm->lock); + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) + num_vmas++; + + return num_vmas; +} + +static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start, + u64 end, struct drm_xe_mem_range_attr *attrs) +{ + struct drm_gpuva *gpuva; + int i = 0; + + lockdep_assert_held(&vm->lock); + + drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + + if (i == *num_vmas) + return -ENOSPC; + + attrs[i].start = xe_vma_start(vma); + attrs[i].end = xe_vma_end(vma); + attrs[i].atomic.val = vma->attr.atomic_access; + attrs[i].pat_index.val = vma->attr.pat_index; + attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd; + attrs[i].preferred_mem_loc.migration_policy = + vma->attr.preferred_loc.migration_policy; + + i++; + } + + *num_vmas = i; + return 0; +} + +int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_mem_range_attr *mem_attrs; + struct drm_xe_vm_query_mem_range_attr *args = data; + u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); + struct xe_vm *vm; + int err = 0; + + if (XE_IOCTL_DBG(xe, + ((args->num_mem_ranges == 0 && + (attrs_user || args->sizeof_mem_range_attr != 0)) || + (args->num_mem_ranges > 0 && + (!attrs_user || + args->sizeof_mem_range_attr != + sizeof(struct drm_xe_mem_range_attr)))))) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + err = down_read_interruptible(&vm->lock); + if (err) + goto put_vm; + + attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); + + if (args->num_mem_ranges == 0 && !attrs_user) { + args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range); + args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr); + goto unlock_vm; + } + + mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr, + GFP_KERNEL | __GFP_ACCOUNT | + __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + if (!mem_attrs) { + err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM; + goto unlock_vm; + } + + memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr); + err = get_mem_attrs(vm, &args->num_mem_ranges, args->start, + args->start + args->range, mem_attrs); + if (err) + goto free_mem_attrs; + + err = copy_to_user(attrs_user, mem_attrs, + args->sizeof_mem_range_attr * args->num_mem_ranges); + +free_mem_attrs: + kvfree(mem_attrs); +unlock_vm: + up_read(&vm->lock); +put_vm: + xe_vm_put(vm); + return err; +} + static bool vma_matches(struct xe_vma *vma, u64 page_addr) { if (page_addr > xe_vma_end(vma) - 1 || diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 015ddb8b8f99..8afd2cd4c9dc 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -199,7 +199,7 @@ int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file); - +int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file); void xe_vm_close_and_put(struct xe_vm *vm); static inline bool xe_vm_in_fault_mode(struct xe_vm *vm) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 115b9bca2a25..7dedd45ab995 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -82,6 +82,7 @@ extern "C" { * - &DRM_IOCTL_XE_WAIT_USER_FENCE * - &DRM_IOCTL_XE_OBSERVATION * - &DRM_IOCTL_XE_MADVISE + * - &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS */ /* @@ -104,6 +105,7 @@ extern "C" { #define DRM_XE_WAIT_USER_FENCE 0x0a #define DRM_XE_OBSERVATION 0x0b #define DRM_XE_MADVISE 0x0c +#define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS 0x0d /* Must be kept compact -- no holes */ @@ -120,6 +122,7 @@ extern "C" { #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param) #define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise) +#define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr) /** * DOC: Xe IOCTL Extensions @@ -2113,6 +2116,143 @@ struct drm_xe_madvise { __u64 reserved[2]; }; +/** + * struct drm_xe_mem_range_attr - Output of &DRM_IOCTL_XE_VM_QUERY_MEM_RANGES_ATTRS + * + * This structure is provided by userspace and filled by KMD in response to the + * DRM_IOCTL_XE_VM_QUERY_MEM_RANGES_ATTRS ioctl. It describes memory attributes of + * a memory ranges within a user specified address range in a VM. + * + * The structure includes information such as atomic access policy, + * page attribute table (PAT) index, and preferred memory location. + * Userspace allocates an array of these structures and passes a pointer to the + * ioctl to retrieve attributes for each memory ranges + * + * @extensions: Pointer to the first extension struct, if any + * @start: Start address of the memory range + * @end: End address of the virtual memory range + * + */ +struct drm_xe_mem_range_attr { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @start: start of the memory range */ + __u64 start; + + /** @end: end of the memory range */ + __u64 end; + + /** @preferred_mem_loc: preferred memory location */ + struct { + /** @preferred_mem_loc.devmem_fd: fd for preferred loc */ + __u32 devmem_fd; + + /** @preferred_mem_loc.migration_policy: Page migration policy */ + __u32 migration_policy; + } preferred_mem_loc; + + /** @atomic: Atomic access policy */ + struct { + /** @atomic.val: atomic attribute */ + __u32 val; + + /** @atomic.reserved: Reserved */ + __u32 reserved; + } atomic; + + /** @pat_index: Page attribute table index */ + struct { + /** @pat_index.val: PAT index */ + __u32 val; + + /** @pat_index.reserved: Reserved */ + __u32 reserved; + } pat_index; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +/** + * struct drm_xe_vm_query_mem_range_attr - Input of &DRM_IOCTL_XE_VM_QUERY_MEM_ATTRIBUTES + * + * This structure is used to query memory attributes of memory regions + * within a user specified address range in a VM. It provides detailed + * information about each memory range, including atomic access policy, + * page attribute table (PAT) index, and preferred memory location. + * + * Userspace first calls the ioctl with @num_mem_ranges = 0, + * @sizeof_mem_ranges_attr = 0 and @vector_of_vma_mem_attr = NULL to retrieve + * the number of memory regions and size of each memory range attribute. + * Then, it allocates a buffer of that size and calls the ioctl again to fill + * the buffer with memory range attributes. + * + * If second call fails with -ENOSPC, it means memory ranges changed between + * first call and now, retry IOCTL again with @num_mem_ranges = 0, + * @sizeof_mem_ranges_attr = 0 and @vector_of_vma_mem_attr = NULL followed by + * Second ioctl call. + * + * Example: + * + * .. code-block:: C + * struct drm_xe_vm_query_mem_range_attr query = { + * .vm_id = vm_id, + * .start = 0x100000, + * .range = 0x2000, + * }; + * + * // First ioctl call to get num of mem regions and sizeof each attribute + * ioctl(fd, DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS, &query); + * + * // Allocate buffer for the memory region attributes + * void *ptr = malloc(query.num_mem_ranges * query.sizeof_mem_range_attr); + * void *ptr_start = ptr; + * + * query.vector_of_mem_attr = (uintptr_t)ptr; + * + * // Second ioctl call to actually fill the memory attributes + * ioctl(fd, DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS, &query); + * + * // Iterate over the returned memory region attributes + * for (unsigned int i = 0; i < query.num_mem_ranges; ++i) { + * struct drm_xe_mem_range_attr *attr = (struct drm_xe_mem_range_attr *)ptr; + * + * // Do something with attr + * + * // Move pointer by one entry + * ptr += query.sizeof_mem_range_attr; + * } + * + * free(ptr_start); + */ +struct drm_xe_vm_query_mem_range_attr { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @vm_id: vm_id of the virtual range */ + __u32 vm_id; + + /** @num_mem_ranges: number of mem_ranges in range */ + __u32 num_mem_ranges; + + /** @start: start of the virtual address range */ + __u64 start; + + /** @range: size of the virtual address range */ + __u64 range; + + /** @sizeof_mem_range_attr: size of struct drm_xe_mem_range_attr */ + __u64 sizeof_mem_range_attr; + + /** @vector_of_mem_attr: userptr to array of struct drm_xe_mem_range_attr */ + __u64 vector_of_mem_attr; + + /** @reserved: Reserved */ + __u64 reserved[2]; + +}; + #if defined(__cplusplus) } #endif From 38fc73b8c7d692a099ddda37b700eeb330a03ff1 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:08 +0530 Subject: [PATCH 142/169] drm/xe: Add documentation for Xe Device Wedging Add documentation for Xe Device Wedging so that file can be referenced in following patches. Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250826063419.3022216-2-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- Documentation/gpu/xe/index.rst | 1 + Documentation/gpu/xe/xe_device.rst | 10 ++++++++++ drivers/gpu/drm/xe/xe_device.c | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 Documentation/gpu/xe/xe_device.rst diff --git a/Documentation/gpu/xe/index.rst b/Documentation/gpu/xe/index.rst index 42ba6c263cd0..88b22fad880e 100644 --- a/Documentation/gpu/xe/index.rst +++ b/Documentation/gpu/xe/index.rst @@ -25,5 +25,6 @@ DG2, etc is provided to prototype the driver. xe_tile xe_debugging xe_devcoredump + xe_device xe-drm-usage-stats.rst xe_configfs diff --git a/Documentation/gpu/xe/xe_device.rst b/Documentation/gpu/xe/xe_device.rst new file mode 100644 index 000000000000..39a937b97cd3 --- /dev/null +++ b/Documentation/gpu/xe/xe_device.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +.. _xe-device-wedging: + +================== +Xe Device Wedging +================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_device.c + :doc: Xe Device Wedging diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 90617e627bc4..6f7f551bd263 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1166,6 +1166,16 @@ static void xe_device_wedged_fini(struct drm_device *drm, void *arg) xe_pm_runtime_put(xe); } +/** + * DOC: Xe Device Wedging + * + * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. + * When device is in wedged state, every IOCTL will be blocked and GT cannot be + * used. Certain critical errors like gt reset failure, firmware failures can cause + * the device to be wedged. The default recovery method for a wedged state + * is rebind/bus-reset. + */ + /** * xe_device_declare_wedged - Declare device wedged * @xe: xe device instance From 9c857a9d84e01332d031b55c4e38a66daecbae73 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:09 +0530 Subject: [PATCH 143/169] drm: Add a vendor-specific recovery method to drm device wedged uevent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the need for a recovery method (firmware flash on Firmware errors) introduced in the later patches of Xe KMD. Whenever XE KMD detects a firmware error, a firmware flash is required to recover the device to normal operation. The initial proposal to use 'firmware-flash' as a recovery method was not applicable to other drivers and could cause multiple recovery methods specific to vendors to be added. To address this a more generic 'vendor-specific' method is introduced, guiding users to refer to vendor specific documentation and system logs for detailed vendor specific recovery procedure. Add a recovery method 'WEDGED=vendor-specific' for such errors. Vendors must provide additional recovery documentation if this method is used. It is the responsibility of the consumer to refer to the correct vendor specific documentation and usecase before attempting a recovery. For example: If driver is XE KMD, the consumer must refer to the documentation of 'Device Wedging' under 'Documentation/gpu/xe/'. v2: fix documentation (Raag) v3: add more details to commit message (Sima, Rodrigo, Raag) add an example script to the documentation (Raag) v4: use consistent naming (Raag) v5: fix commit message v6: add more documentation Cc: André Almeida Cc: Christian König Cc: David Airlie Cc: Simona Vetter Cc: Maxime Ripard Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Acked-by: Maxime Ripard Link: https://lore.kernel.org/r/20250826063419.3022216-3-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- Documentation/gpu/drm-uapi.rst | 47 +++++++++++++++++++++++++++++----- drivers/gpu/drm/drm_drv.c | 2 ++ include/drm/drm_device.h | 4 +++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst index 843facf01b2d..d98428a592f1 100644 --- a/Documentation/gpu/drm-uapi.rst +++ b/Documentation/gpu/drm-uapi.rst @@ -418,13 +418,12 @@ needed. Recovery -------- -Current implementation defines three recovery methods, out of which, drivers +Current implementation defines four recovery methods, out of which, drivers can use any one, multiple or none. Method(s) of choice will be sent in the uevent environment as ``WEDGED=[,..,]`` in order of less to -more side-effects. If driver is unsure about recovery or method is unknown -(like soft/hard system reboot, firmware flashing, physical device replacement -or any other procedure which can't be attempted on the fly), ``WEDGED=unknown`` -will be sent instead. +more side-effects. See the section `Vendor Specific Recovery`_ +for ``WEDGED=vendor-specific``. If driver is unsure about recovery or +method is unknown, ``WEDGED=unknown`` will be sent instead. Userspace consumers can parse this event and attempt recovery as per the following expectations. @@ -435,6 +434,7 @@ following expectations. none optional telemetry collection rebind unbind + bind driver bus-reset unbind + bus reset/re-enumeration + bind + vendor-specific vendor specific recovery method unknown consumer policy =============== ======================================== @@ -446,6 +446,35 @@ telemetry information (devcoredump, syslog). This is useful because the first hang is usually the most critical one which can result in consequential hangs or complete wedging. + +Vendor Specific Recovery +------------------------ + +When ``WEDGED=vendor-specific`` is sent, it indicates that the device requires +a recovery procedure specific to the hardware vendor and is not one of the +standardized approaches. + +``WEDGED=vendor-specific`` may be used to indicate different cases within a +single vendor driver, each requiring a distinct recovery procedure. +In such scenarios, the vendor driver must provide comprehensive documentation +that describes each case, include additional hints to identify specific case and +outline the corresponding recovery procedure. The documentation includes: + +Case - A list of all cases that sends the ``WEDGED=vendor-specific`` recovery method. + +Hints - Additional Information to assist the userspace consumer in identifying and +differentiating between different cases. This can be exposed through sysfs, debugfs, +traces, dmesg etc. + +Recovery Procedure - Clear instructions and guidance for recovering each case. +This may include userspace scripts, tools needed for the recovery procedure. + +It is the responsibility of the admin/userspace consumer to identify the case and +verify additional identification hints before attempting a recovery procedure. + +Example: If the device uses the Xe driver, then userspace consumer should refer to +:ref:`Xe Device Wedging ` for the detailed documentation. + Task information ---------------- @@ -472,8 +501,12 @@ erroring out, all device memory should be unmapped and file descriptors should be closed to prevent leaks or undefined behaviour. The idea here is to clear the device of all user context beforehand and set the stage for a clean recovery. -Example -------- +For ``WEDGED=vendor-specific`` recovery method, it is the responsibility of the +consumer to check the driver documentation and the usecase before attempting +a recovery. + +Example - rebind +---------------- Udev rule:: diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index cdd591b11488..0ac723a46a91 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -532,6 +532,8 @@ static const char *drm_get_wedge_recovery(unsigned int opt) return "rebind"; case DRM_WEDGE_RECOVERY_BUS_RESET: return "bus-reset"; + case DRM_WEDGE_RECOVERY_VENDOR: + return "vendor-specific"; default: return NULL; } diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h index a33aedd5e9ec..59fd3f4d5995 100644 --- a/include/drm/drm_device.h +++ b/include/drm/drm_device.h @@ -26,10 +26,14 @@ struct pci_controller; * Recovery methods for wedged device in order of less to more side-effects. * To be used with drm_dev_wedged_event() as recovery @method. Callers can * use any one, multiple (or'd) or none depending on their needs. + * + * Refer to "Device Wedging" chapter in Documentation/gpu/drm-uapi.rst for more + * details. */ #define DRM_WEDGE_RECOVERY_NONE BIT(0) /* optional telemetry collection */ #define DRM_WEDGE_RECOVERY_REBIND BIT(1) /* unbind + bind driver */ #define DRM_WEDGE_RECOVERY_BUS_RESET BIT(2) /* unbind + reset bus device + bind */ +#define DRM_WEDGE_RECOVERY_VENDOR BIT(3) /* vendor specific recovery method */ /** * struct drm_wedge_task_info - information about the guilty task of a wedge dev From 90fdcf5f89e9288c153923f16a60e6f7da18ba76 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:10 +0530 Subject: [PATCH 144/169] drm/xe: Set GT as wedged before sending wedged uevent Userspace should be notified after setting the device as wedged. Re-order function calls to set gt wedged before sending uevent. Cc: Matthew Brost Suggested-by: Raag Jadav Signed-off-by: Riana Tauro Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250826063419.3022216-4-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 6f7f551bd263..bc8be6fbd2e8 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1180,8 +1180,10 @@ static void xe_device_wedged_fini(struct drm_device *drm, void *arg) * xe_device_declare_wedged - Declare device wedged * @xe: xe device instance * - * This is a final state that can only be cleared with a module + * This is a final state that can only be cleared with the recovery method + * specified in the drm wedged uevent. The default recovery method is * re-probe (unbind + bind). + * * In this state every IOCTL will be blocked so the GT cannot be used. * In general it will be called upon any critical error such as gt reset * failure or guc loading failure. Userspace will be notified of this state @@ -1215,13 +1217,15 @@ void xe_device_declare_wedged(struct xe_device *xe) "IOCTLs and executions are blocked. Only a rebind may clear the failure\n" "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", dev_name(xe->drm.dev)); + } + for_each_gt(gt, xe, id) + xe_gt_declare_wedged(gt); + + if (xe_device_wedged(xe)) { /* Notify userspace of wedged device */ drm_dev_wedged_event(&xe->drm, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET, NULL); } - - for_each_gt(gt, xe, id) - xe_gt_declare_wedged(gt); } From 60439ac3f23549ff272e626bd78502123086eaa5 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:11 +0530 Subject: [PATCH 145/169] drm/xe: Add a helper function to set recovery method Add a helper function to set recovery method. The recovery method can be set before declaring the device wedged and sending the drm wedged uevent. If no method is set, default unbind/re-bind method will be set. v2: fix documentation (Raag) Signed-off-by: Riana Tauro Reviewed-by: Raag Jadav Link: https://lore.kernel.org/r/20250826063419.3022216-5-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 26 +++++++++++++++++++++----- drivers/gpu/drm/xe/xe_device.h | 1 + drivers/gpu/drm/xe/xe_device_types.h | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index bc8be6fbd2e8..ab9f3a57dcd9 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1176,13 +1176,26 @@ static void xe_device_wedged_fini(struct drm_device *drm, void *arg) * is rebind/bus-reset. */ +/** + * xe_device_set_wedged_method - Set wedged recovery method + * @xe: xe device instance + * @method: recovery method to set + * + * Set wedged recovery method to be sent in drm wedged uevent. + */ +void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) +{ + xe->wedged.method = method; +} + /** * xe_device_declare_wedged - Declare device wedged * @xe: xe device instance * * This is a final state that can only be cleared with the recovery method - * specified in the drm wedged uevent. The default recovery method is - * re-probe (unbind + bind). + * specified in the drm wedged uevent. The method can be set using + * xe_device_set_wedged_method before declaring the device as wedged. If no method + * is set, reprobe (unbind/re-bind) will be sent by default. * * In this state every IOCTL will be blocked so the GT cannot be used. * In general it will be called upon any critical error such as gt reset @@ -1223,9 +1236,12 @@ void xe_device_declare_wedged(struct xe_device *xe) xe_gt_declare_wedged(gt); if (xe_device_wedged(xe)) { + /* If no wedge recovery method is set, use default */ + if (!xe->wedged.method) + xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | + DRM_WEDGE_RECOVERY_BUS_RESET); + /* Notify userspace of wedged device */ - drm_dev_wedged_event(&xe->drm, - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET, - NULL); + drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); } } diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index bc802e066a7d..32cc6323b7f6 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -187,6 +187,7 @@ static inline bool xe_device_wedged(struct xe_device *xe) return atomic_read(&xe->wedged.flag); } +void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method); void xe_device_declare_wedged(struct xe_device *xe); struct xe_file *xe_file_get(struct xe_file *xef); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 86506cdfc801..333c7aa6bbb3 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -541,6 +541,8 @@ struct xe_device { atomic_t flag; /** @wedged.mode: Mode controlled by kernel parameter and debugfs */ int mode; + /** @wedged.method: Recovery method to be sent in the drm device wedged uevent */ + unsigned long method; } wedged; /** @bo_device: Struct to control async free of BOs */ From 41ff795aff53613d4184a35bbfae11a15caf07c5 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:12 +0530 Subject: [PATCH 146/169] drm/xe/xe_survivability: Refactor survivability mode Refactor survivability mode code to support both boot and runtime survivability. Signed-off-by: Riana Tauro Reviewed-by: Raag Jadav Link: https://lore.kernel.org/r/20250826063419.3022216-6-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 2 +- drivers/gpu/drm/xe/xe_heci_gsc.c | 2 +- drivers/gpu/drm/xe/xe_pci.c | 6 +- drivers/gpu/drm/xe/xe_survivability_mode.c | 93 +++++++++++++------ drivers/gpu/drm/xe/xe_survivability_mode.h | 4 +- .../gpu/drm/xe/xe_survivability_mode_types.h | 7 ++ 6 files changed, 81 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index ab9f3a57dcd9..d6f10ee6730d 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -743,7 +743,7 @@ int xe_device_probe_early(struct xe_device *xe) * possible, but still return the previous error for error * propagation */ - err = xe_survivability_mode_enable(xe); + err = xe_survivability_mode_boot_enable(xe); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.c b/drivers/gpu/drm/xe/xe_heci_gsc.c index 6d7b62724126..a415ca488791 100644 --- a/drivers/gpu/drm/xe/xe_heci_gsc.c +++ b/drivers/gpu/drm/xe/xe_heci_gsc.c @@ -197,7 +197,7 @@ int xe_heci_gsc_init(struct xe_device *xe) if (ret) return ret; - if (!def->use_polling && !xe_survivability_mode_is_enabled(xe)) { + if (!def->use_polling && !xe_survivability_mode_is_boot_enabled(xe)) { ret = heci_gsc_irq_setup(xe); if (ret) return ret; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index a8bab19111c3..15a863491cdd 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -784,7 +784,7 @@ static void xe_pci_remove(struct pci_dev *pdev) if (IS_SRIOV_PF(xe)) xe_pci_sriov_configure(pdev, 0); - if (xe_survivability_mode_is_enabled(xe)) + if (xe_survivability_mode_is_boot_enabled(xe)) return; xe_device_remove(xe); @@ -866,7 +866,7 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * flashed through mei. Return success, if survivability mode * is enabled due to pcode failure or configfs being set */ - if (xe_survivability_mode_is_enabled(xe)) + if (xe_survivability_mode_is_boot_enabled(xe)) return 0; if (err) @@ -960,7 +960,7 @@ static int xe_pci_suspend(struct device *dev) struct xe_device *xe = pdev_to_xe_device(pdev); int err; - if (xe_survivability_mode_is_enabled(xe)) + if (xe_survivability_mode_is_boot_enabled(xe)) return -EBUSY; err = xe_pm_suspend(xe); diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 41705f5d52e3..4a2d1cff65d2 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -121,6 +121,14 @@ static void log_survivability_info(struct pci_dev *pdev) } } +static int check_boot_failure(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + + return survivability->boot_status == NON_CRITICAL_FAILURE || + survivability->boot_status == CRITICAL_FAILURE; +} + static ssize_t survivability_mode_show(struct device *dev, struct device_attribute *attr, char *buff) { @@ -130,6 +138,11 @@ static ssize_t survivability_mode_show(struct device *dev, struct xe_survivability_info *info = survivability->info; int index = 0, count = 0; + count += sysfs_emit_at(buff, count, "Survivability mode type: Boot\n"); + + if (!check_boot_failure(xe)) + return count; + for (index = 0; index < MAX_SCRATCH_MMIO; index++) { if (info[index].reg) count += sysfs_emit_at(buff, count, "%s: 0x%x - 0x%x\n", info[index].name, @@ -151,12 +164,11 @@ static void xe_survivability_mode_fini(void *arg) sysfs_remove_file(&dev->kobj, &dev_attr_survivability_mode.attr); } -static int enable_survivability_mode(struct pci_dev *pdev) +static int create_survivability_sysfs(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct xe_device *xe = pdev_to_xe_device(pdev); - struct xe_survivability *survivability = &xe->survivability; - int ret = 0; + int ret; /* create survivability mode sysfs */ ret = sysfs_create_file(&dev->kobj, &dev_attr_survivability_mode.attr); @@ -170,6 +182,20 @@ static int enable_survivability_mode(struct pci_dev *pdev) if (ret) return ret; + return 0; +} + +static int enable_boot_survivability_mode(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + int ret = 0; + + ret = create_survivability_sysfs(pdev); + if (ret) + return ret; + /* Make sure xe_heci_gsc_init() knows about survivability mode */ survivability->mode = true; @@ -192,15 +218,36 @@ err: return ret; } +static int init_survivability_mode(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + struct xe_survivability_info *info; + + survivability->size = MAX_SCRATCH_MMIO; + + info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info), + GFP_KERNEL); + if (!info) + return -ENOMEM; + + survivability->info = info; + + populate_survivability_info(xe); + + return 0; +} + /** - * xe_survivability_mode_is_enabled - check if survivability mode is enabled + * xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled * @xe: xe device instance * - * Returns true if in survivability mode, false otherwise + * Returns true if in boot survivability mode of type, else false */ -bool xe_survivability_mode_is_enabled(struct xe_device *xe) +bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe) { - return xe->survivability.mode; + struct xe_survivability *survivability = &xe->survivability; + + return survivability->mode && survivability->type == XE_SURVIVABILITY_TYPE_BOOT; } /** @@ -241,44 +288,38 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe) data = xe_mmio_read32(mmio, PCODE_SCRATCH(0)); survivability->boot_status = REG_FIELD_GET(BOOT_STATUS, data); - return survivability->boot_status == NON_CRITICAL_FAILURE || - survivability->boot_status == CRITICAL_FAILURE; + return check_boot_failure(xe); } /** - * xe_survivability_mode_enable - Initialize and enable the survivability mode + * xe_survivability_mode_boot_enable - Initialize and enable boot survivability mode * @xe: xe device instance * - * Initialize survivability information and enable survivability mode + * Initialize survivability information and enable boot survivability mode * - * Return: 0 if survivability mode is enabled or not requested; negative error + * Return: 0 if boot survivability mode is enabled or not requested, negative error * code otherwise. */ -int xe_survivability_mode_enable(struct xe_device *xe) +int xe_survivability_mode_boot_enable(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; - struct xe_survivability_info *info; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int ret; if (!xe_survivability_mode_is_requested(xe)) return 0; - survivability->size = MAX_SCRATCH_MMIO; + ret = init_survivability_mode(xe); + if (ret) + return ret; - info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info), - GFP_KERNEL); - if (!info) - return -ENOMEM; - - survivability->info = info; - - populate_survivability_info(xe); - - /* Only log debug information and exit if it is a critical failure */ + /* Log breadcrumbs but do not enter survivability mode for Critical boot errors */ if (survivability->boot_status == CRITICAL_FAILURE) { log_survivability_info(pdev); return -ENXIO; } - return enable_survivability_mode(pdev); + survivability->type = XE_SURVIVABILITY_TYPE_BOOT; + + return enable_boot_survivability_mode(pdev); } diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h index 02231c2bf008..f6ee283ea5e8 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -10,8 +10,8 @@ struct xe_device; -int xe_survivability_mode_enable(struct xe_device *xe); -bool xe_survivability_mode_is_enabled(struct xe_device *xe); +int xe_survivability_mode_boot_enable(struct xe_device *xe); +bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); bool xe_survivability_mode_is_requested(struct xe_device *xe); #endif /* _XE_SURVIVABILITY_MODE_H_ */ diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h index 19d433e253df..5dce393498da 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode_types.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h @@ -9,6 +9,10 @@ #include #include +enum xe_survivability_type { + XE_SURVIVABILITY_TYPE_BOOT, +}; + struct xe_survivability_info { char name[NAME_MAX]; u32 reg; @@ -30,6 +34,9 @@ struct xe_survivability { /** @mode: boolean to indicate survivability mode */ bool mode; + + /** @type: survivability type */ + enum xe_survivability_type type; }; #endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */ From a2ca0633a0fef925a0d8125d8f3e4495a5ecb310 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:13 +0530 Subject: [PATCH 147/169] drm/xe/xe_survivability: Add support for Runtime survivability mode Certain runtime firmware errors can cause the device to be in a unusable state requiring a firmware flash to restore normal operation. Runtime Survivability Mode indicates firmware flash is necessary by wedging the device and exposing survivability mode sysfs. The below sysfs is an indication that device is in survivability mode /sys/bus/pci/devices//survivability_mode v2: Fix kernel-doc (Umesh) v3: Add user friendly dmesg (Frank) Signed-off-by: Riana Tauro Reviewed-by: Raag Jadav Link: https://lore.kernel.org/r/20250826063419.3022216-7-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_survivability_mode.c | 43 ++++++++++++++++++- drivers/gpu/drm/xe/xe_survivability_mode.h | 1 + .../gpu/drm/xe/xe_survivability_mode_types.h | 1 + 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 4a2d1cff65d2..267d0e3fd85a 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -138,7 +138,8 @@ static ssize_t survivability_mode_show(struct device *dev, struct xe_survivability_info *info = survivability->info; int index = 0, count = 0; - count += sysfs_emit_at(buff, count, "Survivability mode type: Boot\n"); + count += sysfs_emit_at(buff, count, "Survivability mode type: %s\n", + survivability->type ? "Runtime" : "Boot"); if (!check_boot_failure(xe)) return count; @@ -291,6 +292,46 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe) return check_boot_failure(xe); } +/** + * xe_survivability_mode_runtime_enable - Initialize and enable runtime survivability mode + * @xe: xe device instance + * + * Initialize survivability information and enable runtime survivability mode. + * Runtime survivability mode is enabled when certain errors cause the device to be + * in non-recoverable state. The device is declared wedged with the appropriate + * recovery method and survivability mode sysfs exposed to userspace + * + * Return: 0 if runtime survivability mode is enabled, negative error code otherwise. + */ +int xe_survivability_mode_runtime_enable(struct xe_device *xe) +{ + struct xe_survivability *survivability = &xe->survivability; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int ret; + + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) { + dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n"); + return -EINVAL; + } + + ret = init_survivability_mode(xe); + if (ret) + return ret; + + ret = create_survivability_sysfs(pdev); + if (ret) + dev_err(&pdev->dev, "Failed to create survivability mode sysfs\n"); + + survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; + dev_err(&pdev->dev, "Runtime Survivability mode enabled\n"); + + xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); + xe_device_declare_wedged(xe); + dev_err(&pdev->dev, "Firmware update required, Refer the userspace documentation for more details!\n"); + + return 0; +} + /** * xe_survivability_mode_boot_enable - Initialize and enable boot survivability mode * @xe: xe device instance diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h index f6ee283ea5e8..1cc94226aa82 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -11,6 +11,7 @@ struct xe_device; int xe_survivability_mode_boot_enable(struct xe_device *xe); +int xe_survivability_mode_runtime_enable(struct xe_device *xe); bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); bool xe_survivability_mode_is_requested(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h index 5dce393498da..cd65a5d167c9 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode_types.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h @@ -11,6 +11,7 @@ enum xe_survivability_type { XE_SURVIVABILITY_TYPE_BOOT, + XE_SURVIVABILITY_TYPE_RUNTIME, }; struct xe_survivability_info { From f646c9f9371b28b8f93e619fe003415f6aaeb416 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:14 +0530 Subject: [PATCH 148/169] drm/xe/doc: Document device wedged and runtime survivability Add documentation for vendor specific device wedged recovery method and runtime survivability. v2: fix documentation (Raag) v3: add userspace tool for firmware update (Raag) v4: use consistent documentation (Raag) v5: add more documentation Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Reviewed-by: Raag Jadav Link: https://lore.kernel.org/r/20250826063419.3022216-8-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- Documentation/gpu/xe/xe_pcode.rst | 6 ++-- drivers/gpu/drm/xe/xe_device.c | 27 +++++++++++++++++ drivers/gpu/drm/xe/xe_survivability_mode.c | 35 +++++++++++++++++----- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/Documentation/gpu/xe/xe_pcode.rst b/Documentation/gpu/xe/xe_pcode.rst index 5937ef3599b0..2a43601123cb 100644 --- a/Documentation/gpu/xe/xe_pcode.rst +++ b/Documentation/gpu/xe/xe_pcode.rst @@ -13,9 +13,11 @@ Internal API .. kernel-doc:: drivers/gpu/drm/xe/xe_pcode.c :internal: +.. _xe-survivability-mode: + ================== -Boot Survivability +Survivability Mode ================== .. kernel-doc:: drivers/gpu/drm/xe/xe_survivability_mode.c - :doc: Xe Boot Survivability + :doc: Survivability Mode diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index d6f10ee6730d..9e4773a17ef8 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1174,6 +1174,33 @@ static void xe_device_wedged_fini(struct drm_device *drm, void *arg) * used. Certain critical errors like gt reset failure, firmware failures can cause * the device to be wedged. The default recovery method for a wedged state * is rebind/bus-reset. + * + * Another recovery method is vendor-specific. Below are the cases that send + * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. + * + * Case: Firmware Flash + * -------------------- + * + * Identification Hint + * +++++++++++++++++++ + * + * ``WEDGED=vendor-specific`` drm device wedged uevent with + * :ref:`Runtime Survivability mode ` is used to notify + * admin/userspace consumer about the need for a firmware flash. + * + * Recovery Procedure + * ++++++++++++++++++ + * + * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow + * the below steps + * + * - Check Runtime Survivability mode sysfs. + * If enabled, firmware flash is required to recover the device. + * + * /sys/bus/pci/devices//survivability_mode + * + * - Admin/userpsace consumer can use firmware flashing tools like fwupd to flash + * firmware and restore device to normal operation. */ /** diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 267d0e3fd85a..86ba767c4e44 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -22,15 +22,18 @@ #define MAX_SCRATCH_MMIO 8 /** - * DOC: Xe Boot Survivability + * DOC: Survivability Mode * - * Boot Survivability is a software based workflow for recovering a system in a failed boot state + * Survivability Mode is a software based workflow for recovering a system in a failed boot state * Here system recoverability is concerned with recovering the firmware responsible for boot. * - * This is implemented by loading the driver with bare minimum (no drm card) to allow the firmware - * to be flashed through mei and collect telemetry. The driver's probe flow is modified - * such that it enters survivability mode when pcode initialization is incomplete and boot status - * denotes a failure. + * Boot Survivability + * =================== + * + * Boot Survivability is implemented by loading the driver with bare minimum (no drm card) to allow + * the firmware to be flashed through mei driver and collect telemetry. The driver's probe flow is + * modified such that it enters survivability mode when pcode initialization is incomplete and boot + * status denotes a failure. * * Survivability mode can also be entered manually using the survivability mode attribute available * through configfs which is beneficial in several usecases. It can be used to address scenarios @@ -46,7 +49,7 @@ * Survivability mode is indicated by the below admin-only readable sysfs which provides additional * debug information:: * - * /sys/bus/pci/devices//surivability_mode + * /sys/bus/pci/devices//survivability_mode * * Capability Information: * Provides boot status @@ -56,6 +59,22 @@ * Provides history of previous failures * Auxiliary Information * Certain failures may have information in addition to postcode information + * + * Runtime Survivability + * ===================== + * + * Certain runtime firmware errors can cause the device to enter a wedged state + * (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation. + * Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and + * is indicated by the presence of survivability mode sysfs:: + * + * /sys/bus/pci/devices//survivability_mode + * + * Survivability mode sysfs provides information about the type of survivability mode. + * + * When such errors occur, userspace is notified with the drm device wedged uevent and runtime + * survivability mode. User can then initiate a firmware flash using userspace tools like fwupd + * to restore device to normal operation. */ static u32 aux_history_offset(u32 reg_value) @@ -327,7 +346,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_device_declare_wedged(xe); - dev_err(&pdev->dev, "Firmware update required, Refer the userspace documentation for more details!\n"); + dev_err(&pdev->dev, "Firmware flash required, Refer the userspace documentation for more details!\n"); return 0; } From 0a2a873d615a39e8a87d3f15285ed888341ddce8 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:15 +0530 Subject: [PATCH 149/169] drm/xe: Add support to handle hardware errors Gfx device reports two classes of errors: uncorrectable and correctable. Depending on the severity uncorrectable errors are further classified Non-Fatal and Fatal. Correctable and Non-Fatal errors: These errors are reported as MSI. Bits in the Master Interrupt Register indicate the class of the error. The source of the error is then read from the Device Error Source Register. Fatal errors: These are reported as PCIe errors When a PCIe error is asserted, the OS will perform a SBR (Secondary Bus reset) which causes the driver to reload. The error registers are sticky and the values are maintained through SBR. Add basic support to handle these errors. Bspec: 50875, 53073, 53074, 53075, 53076 v2: Format commit message (Umesh) v3: fix documentation (Stuart) Cc: Stuart Summers Co-developed-by: Himal Prasad Ghimiray Signed-off-by: Himal Prasad Ghimiray Signed-off-by: Riana Tauro Reviewed-by: Umesh Nerlige Ramappa Link: https://lore.kernel.org/r/20250826063419.3022216-9-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/regs/xe_hw_error_regs.h | 15 +++ drivers/gpu/drm/xe/regs/xe_irq_regs.h | 1 + drivers/gpu/drm/xe/xe_hw_error.c | 109 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_hw_error.h | 15 +++ drivers/gpu/drm/xe/xe_irq.c | 4 + 6 files changed, 145 insertions(+) create mode 100644 drivers/gpu/drm/xe/regs/xe_hw_error_regs.h create mode 100644 drivers/gpu/drm/xe/xe_hw_error.c create mode 100644 drivers/gpu/drm/xe/xe_hw_error.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index c52cac979e05..0abd8078cfb0 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -82,6 +82,7 @@ xe-y += xe_bb.o \ xe_hw_engine.o \ xe_hw_engine_class_sysfs.o \ xe_hw_engine_group.o \ + xe_hw_error.o \ xe_hw_fence.o \ xe_irq.o \ xe_lrc.o \ diff --git a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h new file mode 100644 index 000000000000..ed9b81fb28a0 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_HW_ERROR_REGS_H_ +#define _XE_HW_ERROR_REGS_H_ + +#define DEV_ERR_STAT_NONFATAL 0x100178 +#define DEV_ERR_STAT_CORRECTABLE 0x10017c +#define DEV_ERR_STAT_REG(x) XE_REG(_PICK_EVEN((x), \ + DEV_ERR_STAT_CORRECTABLE, \ + DEV_ERR_STAT_NONFATAL)) + +#endif diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h index 13635e4331d4..7c2a3a140142 100644 --- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h @@ -18,6 +18,7 @@ #define GFX_MSTR_IRQ XE_REG(0x190010, XE_REG_OPTION_VF) #define MASTER_IRQ REG_BIT(31) #define GU_MISC_IRQ REG_BIT(29) +#define ERROR_IRQ(x) REG_BIT(26 + (x)) #define DISPLAY_IRQ REG_BIT(16) #define I2C_IRQ REG_BIT(12) #define GT_DW_IRQ(x) REG_BIT(x) diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c new file mode 100644 index 000000000000..0fa45792f1c0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "regs/xe_hw_error_regs.h" +#include "regs/xe_irq_regs.h" + +#include "xe_device.h" +#include "xe_hw_error.h" +#include "xe_mmio.h" + +/* Error categories reported by hardware */ +enum hardware_error { + HARDWARE_ERROR_CORRECTABLE = 0, + HARDWARE_ERROR_NONFATAL = 1, + HARDWARE_ERROR_FATAL = 2, + HARDWARE_ERROR_MAX, +}; + +static const char *hw_error_to_str(const enum hardware_error hw_err) +{ + switch (hw_err) { + case HARDWARE_ERROR_CORRECTABLE: + return "CORRECTABLE"; + case HARDWARE_ERROR_NONFATAL: + return "NONFATAL"; + case HARDWARE_ERROR_FATAL: + return "FATAL"; + default: + return "UNKNOWN"; + } +} + +static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_error hw_err) +{ + const char *hw_err_str = hw_error_to_str(hw_err); + struct xe_device *xe = tile_to_xe(tile); + unsigned long flags; + u32 err_src; + + if (xe->info.platform != XE_BATTLEMAGE) + return; + + spin_lock_irqsave(&xe->irq.lock, flags); + err_src = xe_mmio_read32(&tile->mmio, DEV_ERR_STAT_REG(hw_err)); + if (!err_src) { + drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported DEV_ERR_STAT_%s blank!\n", + tile->id, hw_err_str); + goto unlock; + } + + /* TODO: Process errrors per source */ + + xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src); + +unlock: + spin_unlock_irqrestore(&xe->irq.lock, flags); +} + +/** + * xe_hw_error_irq_handler - irq handling for hw errors + * @tile: tile instance + * @master_ctl: value read from master interrupt register + * + * Xe platforms add three error bits to the master interrupt register to support error handling. + * These three bits are used to convey the class of error FATAL, NONFATAL, or CORRECTABLE. + * To process the interrupt, determine the source of error by reading the Device Error Source + * Register that corresponds to the class of error being serviced. + */ +void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl) +{ + enum hardware_error hw_err; + + for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) + if (master_ctl & ERROR_IRQ(hw_err)) + hw_error_source_handler(tile, hw_err); +} + +/* + * Process hardware errors during boot + */ +static void process_hw_errors(struct xe_device *xe) +{ + struct xe_tile *tile; + u32 master_ctl; + u8 id; + + for_each_tile(tile, xe, id) { + master_ctl = xe_mmio_read32(&tile->mmio, GFX_MSTR_IRQ); + xe_hw_error_irq_handler(tile, master_ctl); + xe_mmio_write32(&tile->mmio, GFX_MSTR_IRQ, master_ctl); + } +} + +/** + * xe_hw_error_init - Initialize hw errors + * @xe: xe device instance + * + * Initialize and check for errors that occurred during boot + * prior to driver load + */ +void xe_hw_error_init(struct xe_device *xe) +{ + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) + return; + + process_hw_errors(xe); +} diff --git a/drivers/gpu/drm/xe/xe_hw_error.h b/drivers/gpu/drm/xe/xe_hw_error.h new file mode 100644 index 000000000000..d86e28c5180c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_error.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ +#ifndef XE_HW_ERROR_H_ +#define XE_HW_ERROR_H_ + +#include + +struct xe_tile; +struct xe_device; + +void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl); +void xe_hw_error_init(struct xe_device *xe); +#endif diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index 5df5b8c2a3e4..870edaf69388 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -18,6 +18,7 @@ #include "xe_gt.h" #include "xe_guc.h" #include "xe_hw_engine.h" +#include "xe_hw_error.h" #include "xe_i2c.h" #include "xe_memirq.h" #include "xe_mmio.h" @@ -468,6 +469,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl); gt_irq_handler(tile, master_ctl, intr_dw, identity); + xe_hw_error_irq_handler(tile, master_ctl); /* * Display interrupts (including display backlight operations @@ -756,6 +758,8 @@ int xe_irq_install(struct xe_device *xe) int nvec = 1; int err; + xe_hw_error_init(xe); + xe_irq_reset(xe); if (xe_device_has_msix(xe)) { From a7df563b45b03ac8aa480051cce8d49d7b489ec6 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:16 +0530 Subject: [PATCH 150/169] drm/xe/xe_hw_error: Handle CSC Firmware reported Hardware errors Add support to handle CSC firmware reported errors. When CSC firmware errors are encoutered, a error interrupt is received by the GFX device as a MSI interrupt. Device Source control registers indicates the source of the error as CSC The HEC error status register indicates that the error is firmware reported Depending on the type of error, the error cause is written to the HEC Firmware error register. On encountering such CSC firmware errors, the graphics device is non-recoverable from driver context. The only way to recover from these errors is firmware flash. System admin/userspace is notified of the necessity of firmware flash with a combination of vendor-specific drm device edged uevent, dmesg logs and runtime survivability sysfs. It is the responsiblity of the consumer to verify all the actions and then trigger a firmware flash using tools like fwupd. $ udevadm monitor --property --kernel monitor will print the received events for: KERNEL - the kernel uevent KERNEL[754.709341] change /devices/pci0000:00/0000:00:01.0/0000:01:00.0/0000:02:01.0/0000:03:00.0/drm/card0 (drm) ACTION=change DEVPATH=/devices/pci0000:00/0000:00:01.0/0000:01:00.0/0000:02:01.0/0000:03:00.0/drm/card0 SUBSYSTEM=drm WEDGED=vendor-specific DEVNAME=/dev/dri/card0 DEVTYPE=drm_minor SEQNUM=5973 MAJOR=226 MINOR=0 Logs xe 0000:03:00.0: [drm] *ERROR* [Hardware Error]: Tile0 reported NONFATAL error 0x20000 xe 0000:03:00.0: [drm] *ERROR* [Hardware Error]: NONFATAL: HEC Uncorrected FW FD Corruption error reported, bit[2] is set xe 0000:03:00.0: Runtime Survivability mode enabled xe 0000:03:00.0: [drm] *ERROR* CRITICAL: Xe has declared device 0000:03:00.0 as wedged. IOCTLs and executions are blocked. Only a rebind may clear the failure Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new xe 0000:03:00.0: [drm] device wedged, needs recovery xe 0000:03:00.0: Firmware flash required, Please refer to the userspace documentation for more details! Runtime survivability Sysfs: /sys/bus/pci/devices//survivability_mode v2: use vendor recovery method with runtime survivability (Christian, Rodrigo, Raag) v3: move declare wedged to runtime survivability mode (Rodrigo) v4: update commit message Signed-off-by: Riana Tauro Reviewed-by: Umesh Nerlige Ramappa Link: https://lore.kernel.org/r/20250826063419.3022216-10-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/regs/xe_gsc_regs.h | 2 + drivers/gpu/drm/xe/regs/xe_hw_error_regs.h | 7 ++- drivers/gpu/drm/xe/xe_device_types.h | 3 + drivers/gpu/drm/xe/xe_hw_error.c | 64 +++++++++++++++++++++- drivers/gpu/drm/xe/xe_survivability_mode.c | 2 +- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h index 9b66cc972a63..180be82672ab 100644 --- a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h @@ -13,6 +13,8 @@ /* Definitions of GSC H/W registers, bits, etc */ +#define BMG_GSC_HECI1_BASE 0x373000 + #define MTL_GSC_HECI1_BASE 0x00116000 #define MTL_GSC_HECI2_BASE 0x00117000 diff --git a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h index ed9b81fb28a0..c146b9ef44eb 100644 --- a/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_hw_error_regs.h @@ -6,10 +6,15 @@ #ifndef _XE_HW_ERROR_REGS_H_ #define _XE_HW_ERROR_REGS_H_ +#define HEC_UNCORR_ERR_STATUS(base) XE_REG((base) + 0x118) +#define UNCORR_FW_REPORTED_ERR BIT(6) + +#define HEC_UNCORR_FW_ERR_DW0(base) XE_REG((base) + 0x124) + #define DEV_ERR_STAT_NONFATAL 0x100178 #define DEV_ERR_STAT_CORRECTABLE 0x10017c #define DEV_ERR_STAT_REG(x) XE_REG(_PICK_EVEN((x), \ DEV_ERR_STAT_CORRECTABLE, \ DEV_ERR_STAT_NONFATAL)) - +#define XE_CSC_ERROR BIT(17) #endif diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 333c7aa6bbb3..389996970564 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -192,6 +192,9 @@ struct xe_tile { /** @memirq: Memory Based Interrupts. */ struct xe_memirq memirq; + /** @csc_hw_error_work: worker to report CSC HW errors */ + struct work_struct csc_hw_error_work; + /** @pcode: tile's PCODE */ struct { /** @pcode.lock: protecting tile's PCODE mailbox data */ diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 0fa45792f1c0..4099ed1262e1 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -3,12 +3,16 @@ * Copyright © 2025 Intel Corporation */ +#include "regs/xe_gsc_regs.h" #include "regs/xe_hw_error_regs.h" #include "regs/xe_irq_regs.h" #include "xe_device.h" #include "xe_hw_error.h" #include "xe_mmio.h" +#include "xe_survivability_mode.h" + +#define HEC_UNCORR_FW_ERR_BITS 4 /* Error categories reported by hardware */ enum hardware_error { @@ -18,6 +22,13 @@ enum hardware_error { HARDWARE_ERROR_MAX, }; +static const char * const hec_uncorrected_fw_errors[] = { + "Fatal", + "CSE Disabled", + "FD Corruption", + "Data Corruption" +}; + static const char *hw_error_to_str(const enum hardware_error hw_err) { switch (hw_err) { @@ -32,6 +43,52 @@ static const char *hw_error_to_str(const enum hardware_error hw_err) } } +static void csc_hw_error_work(struct work_struct *work) +{ + struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); + struct xe_device *xe = tile_to_xe(tile); + int ret; + + ret = xe_survivability_mode_runtime_enable(xe); + if (ret) + drm_err(&xe->drm, "Failed to enable runtime survivability mode\n"); +} + +static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err) +{ + const char *hw_err_str = hw_error_to_str(hw_err); + struct xe_device *xe = tile_to_xe(tile); + struct xe_mmio *mmio = &tile->mmio; + u32 base, err_bit, err_src; + unsigned long fw_err; + + if (xe->info.platform != XE_BATTLEMAGE) + return; + + base = BMG_GSC_HECI1_BASE; + lockdep_assert_held(&xe->irq.lock); + err_src = xe_mmio_read32(mmio, HEC_UNCORR_ERR_STATUS(base)); + if (!err_src) { + drm_err_ratelimited(&xe->drm, HW_ERR "Tile%d reported HEC_ERR_STATUS_%s blank\n", + tile->id, hw_err_str); + return; + } + + if (err_src & UNCORR_FW_REPORTED_ERR) { + fw_err = xe_mmio_read32(mmio, HEC_UNCORR_FW_ERR_DW0(base)); + for_each_set_bit(err_bit, &fw_err, HEC_UNCORR_FW_ERR_BITS) { + drm_err_ratelimited(&xe->drm, HW_ERR + "%s: HEC Uncorrected FW %s error reported, bit[%d] is set\n", + hw_err_str, hec_uncorrected_fw_errors[err_bit], + err_bit); + + schedule_work(&tile->csc_hw_error_work); + } + } + + xe_mmio_write32(mmio, HEC_UNCORR_ERR_STATUS(base), err_src); +} + static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_error hw_err) { const char *hw_err_str = hw_error_to_str(hw_err); @@ -50,7 +107,8 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er goto unlock; } - /* TODO: Process errrors per source */ + if (err_src & XE_CSC_ERROR) + csc_hw_error_handler(tile, hw_err); xe_mmio_write32(&tile->mmio, DEV_ERR_STAT_REG(hw_err), err_src); @@ -102,8 +160,12 @@ static void process_hw_errors(struct xe_device *xe) */ void xe_hw_error_init(struct xe_device *xe) { + struct xe_tile *tile = xe_device_get_root_tile(xe); + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) return; + INIT_WORK(&tile->csc_hw_error_work, csc_hw_error_work); + process_hw_errors(xe); } diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 86ba767c4e44..53c5af4b810c 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -346,7 +346,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_device_declare_wedged(xe); - dev_err(&pdev->dev, "Firmware flash required, Refer the userspace documentation for more details!\n"); + dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n"); return 0; } From d1f51a4f953da4ada15fa585c800ca98d627ce47 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Tue, 26 Aug 2025 12:04:17 +0530 Subject: [PATCH 151/169] drm/xe/xe_hw_error: Add fault injection to trigger csc error handler Add a debugfs fault handler to trigger csc error handler that wedges the device and enables runtime survivability mode. v2: add debugfs only for bmg (Umesh) v3: do not use csc_fault attribute if debugfs is not enabled v4: rebase Cc: Lucas De Marchi Signed-off-by: Riana Tauro Reviewed-by: Raag Jadav Link: https://lore.kernel.org/r/20250826063419.3022216-11-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_debugfs.c | 6 +++++- drivers/gpu/drm/xe/xe_hw_error.c | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index bc717519502d..8d6df6bd885e 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -35,6 +35,7 @@ #endif DECLARE_FAULT_ATTR(gt_reset_failure); +DECLARE_FAULT_ATTR(inject_csc_hw_error); static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio, u32 offset, char *name, struct drm_printer *p) @@ -361,10 +362,13 @@ void xe_debugfs_register(struct xe_device *xe) ARRAY_SIZE(debugfs_list), root, minor); - if (xe->info.platform == XE_BATTLEMAGE) + if (xe->info.platform == XE_BATTLEMAGE) { drm_debugfs_create_files(debugfs_residencies, ARRAY_SIZE(debugfs_residencies), root, minor); + fault_create_debugfs_attr("inject_csc_hw_error", root, + &inject_csc_hw_error); + } debugfs_create_file("forcewake_all", 0400, root, xe, &forcewake_all_fops); diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 4099ed1262e1..8c65291f36fc 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -3,6 +3,8 @@ * Copyright © 2025 Intel Corporation */ +#include + #include "regs/xe_gsc_regs.h" #include "regs/xe_hw_error_regs.h" #include "regs/xe_irq_regs.h" @@ -13,6 +15,7 @@ #include "xe_survivability_mode.h" #define HEC_UNCORR_FW_ERR_BITS 4 +extern struct fault_attr inject_csc_hw_error; /* Error categories reported by hardware */ enum hardware_error { @@ -43,6 +46,11 @@ static const char *hw_error_to_str(const enum hardware_error hw_err) } } +static bool fault_inject_csc_hw_error(void) +{ + return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(&inject_csc_hw_error, 1); +} + static void csc_hw_error_work(struct work_struct *work) { struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); @@ -130,6 +138,9 @@ void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl) { enum hardware_error hw_err; + if (fault_inject_csc_hw_error()) + schedule_work(&tile->csc_hw_error_work); + for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) if (master_ctl & ERROR_IRQ(hw_err)) hw_error_source_handler(tile, hw_err); From d738e1be2b2b4364403babc43ae7343d45e99d41 Mon Sep 17 00:00:00 2001 From: Chaitanya Kumar Borah Date: Fri, 22 Aug 2025 05:55:12 +0530 Subject: [PATCH 152/169] drm/xe/wcl: Extend L3bank mask workaround The commit 9ab440a9d042 ("drm/xe/ptl: L3bank mask is not available on the media GT") added a workaround to ignore the fuse register that L3 bank availability as it did not contain valid values. Same is true for WCL therefore extend the workaround to cover it. Signed-off-by: Chaitanya Kumar Borah Reviewed-by: Dnyaneshwar Bhadane Link: https://lore.kernel.org/r/20250822002512.1129144-1-chaitanya.kumar.borah@intel.com Signed-off-by: Gustavo Sousa --- drivers/gpu/drm/xe/xe_wa_oob.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 27cd19b98f8e..0cf2b3c03532 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -48,7 +48,7 @@ 16023588340 GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_not_sriov_vf) 14019789679 GRAPHICS_VERSION(1255) GRAPHICS_VERSION_RANGE(1270, 2004) -no_media_l3 MEDIA_VERSION(3000) +no_media_l3 MEDIA_VERSION_RANGE(3000, 3002) 14022866841 GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0) MEDIA_VERSION(3000), MEDIA_STEP(A0, B0) 16021333562 GRAPHICS_VERSION_RANGE(1200, 1274) From 1313351e71181a4818afeb8dfe202e4162091ef6 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 26 Aug 2025 17:06:32 -0700 Subject: [PATCH 153/169] drm/xe: make xe_gt_idle_disable_c6() handle the forcewake internally Move forcewake_get() into xe_gt_idle_enable_c6() to streamline the code and make it easier to use. Suggested-by: Rodrigo Vivi Signed-off-by: Xin Wang Reviewed-by: Rodrigo Vivi Link: https://lore.kernel.org/r/20250827000633.1369890-2-x.wang@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_gt_idle.c | 21 +++++++++++++-------- drivers/gpu/drm/xe/xe_gt_idle.h | 2 +- drivers/gpu/drm/xe/xe_guc_pc.c | 13 +------------ 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index ffb210216aa9..f8950a52d0a4 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -322,15 +322,11 @@ static void gt_idle_fini(void *arg) { struct kobject *kobj = arg; struct xe_gt *gt = kobj_to_gt(kobj->parent); - unsigned int fw_ref; xe_gt_idle_disable_pg(gt); - if (gt_to_xe(gt)->info.skip_guc_pc) { - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (gt_to_xe(gt)->info.skip_guc_pc) xe_gt_idle_disable_c6(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - } sysfs_remove_files(kobj, gt_idle_attrs); kobject_put(kobj); @@ -390,14 +386,23 @@ void xe_gt_idle_enable_c6(struct xe_gt *gt) RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE); } -void xe_gt_idle_disable_c6(struct xe_gt *gt) +int xe_gt_idle_disable_c6(struct xe_gt *gt) { + unsigned int fw_ref; + xe_device_assert_mem_access(gt_to_xe(gt)); - xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); if (IS_SRIOV_VF(gt_to_xe(gt))) - return; + return 0; + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref) + return -ETIMEDOUT; xe_mmio_write32(>->mmio, RC_CONTROL, 0); xe_mmio_write32(>->mmio, RC_STATE, 0); + + xe_force_wake_put(gt_to_fw(gt), fw_ref); + + return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_idle.h b/drivers/gpu/drm/xe/xe_gt_idle.h index 591a01e181bc..9c34a155e102 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.h +++ b/drivers/gpu/drm/xe/xe_gt_idle.h @@ -13,7 +13,7 @@ struct xe_gt; int xe_gt_idle_init(struct xe_gt_idle *gtidle); void xe_gt_idle_enable_c6(struct xe_gt *gt); -void xe_gt_idle_disable_c6(struct xe_gt *gt); +int xe_gt_idle_disable_c6(struct xe_gt *gt); void xe_gt_idle_enable_pg(struct xe_gt *gt); void xe_gt_idle_disable_pg(struct xe_gt *gt); int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p); diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 0f8430acd16d..88557e86d637 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1076,7 +1076,6 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc) { struct xe_device *xe = pc_to_xe(pc); struct xe_gt *gt = pc_to_gt(pc); - unsigned int fw_ref; int ret = 0; if (xe->info.skip_guc_pc) @@ -1086,17 +1085,7 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc) if (ret) return ret; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return -ETIMEDOUT; - } - - xe_gt_idle_disable_c6(gt); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); - - return 0; + return xe_gt_idle_disable_c6(gt); } /** From 95d0883ac8105717f59c2dcdc0d8b9150f13aa12 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 26 Aug 2025 17:06:33 -0700 Subject: [PATCH 154/169] drm/xe: Ensure GT is in C0 during resumes This patch ensures the gt will be awake for the entire duration of the resume sequences until GuCRC takes over and GT-C6 gets re-enabled. Before suspending GT-C6 is kept enabled, but upon resume, GuCRC is not yet alive to properly control the exits and some cases of instability and corruption related to GT-C6 can be observed. Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/4037 Suggested-by: Rodrigo Vivi Signed-off-by: Xin Wang Reviewed-by: Rodrigo Vivi Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/4037 Link: https://lore.kernel.org/r/20250827000633.1369890-3-x.wang@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pm.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 51ed7270994a..d4b54cd34cff 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -18,7 +18,7 @@ #include "xe_device.h" #include "xe_ggtt.h" #include "xe_gt.h" -#include "xe_guc.h" +#include "xe_gt_idle.h" #include "xe_i2c.h" #include "xe_irq.h" #include "xe_pcode.h" @@ -177,6 +177,9 @@ int xe_pm_resume(struct xe_device *xe) drm_dbg(&xe->drm, "Resuming device\n"); trace_xe_pm_resume(xe, __builtin_return_address(0)); + for_each_gt(gt, xe, id) + xe_gt_idle_disable_c6(gt); + for_each_tile(tile, xe, id) xe_wa_apply_tile_workarounds(tile); @@ -537,6 +540,9 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_rpm_lockmap_acquire(xe); + for_each_gt(gt, xe, id) + xe_gt_idle_disable_c6(gt); + if (xe->d3cold.allowed) { err = xe_pcode_ready(xe, true); if (err) From 2674f1ef29f46ae81464d0ac44b7824deb9de85c Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Tue, 26 Aug 2025 08:32:11 -0700 Subject: [PATCH 155/169] drm/xe/configfs: Block runtime attribute changes Although it's possible to change the attributes in runtime, they have no effect after the driver is already bound to the device. Check for that and return -EBUSY in that case. This should help users understand what's going on when the behavior is not changing even if the value from the configfs is "right", but it got to that state too late. Reviewed-by: Riana Tauro Link: https://lore.kernel.org/r/20250826153210.3068808-2-lucas.demarchi@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_configfs.c | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 29a12d02443d..1025d3979b06 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -75,6 +75,8 @@ * * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/survivability_mode * + * This attribute can only be set before binding to the device. + * * Allowed engines: * ---------------- * @@ -99,6 +101,8 @@ * available for migrations, but it's disabled. This is intended for debugging * purposes only. * + * This attribute can only be set before binding to the device. + * * PSMI * ---- * @@ -109,6 +113,8 @@ * * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/enable_psmi * + * This attribute can only be set before binding to the device. + * * Remove devices * ============== * @@ -169,6 +175,32 @@ static struct xe_config_device *to_xe_config_device(struct config_item *item) return &to_xe_config_group_device(item)->config; } +static bool is_bound(struct xe_config_group_device *dev) +{ + unsigned int domain, bus, slot, function; + struct pci_dev *pdev; + const char *name; + bool ret; + + lockdep_assert_held(&dev->lock); + + name = dev->group.cg_item.ci_name; + if (sscanf(name, "%x:%x:%x.%x", &domain, &bus, &slot, &function) != 4) + return false; + + pdev = pci_get_domain_bus_and_slot(domain, bus, PCI_DEVFN(slot, function)); + if (!pdev) + return false; + + ret = pci_get_drvdata(pdev); + pci_dev_put(pdev); + + if (ret) + pci_dbg(pdev, "Already bound to driver\n"); + + return ret; +} + static ssize_t survivability_mode_show(struct config_item *item, char *page) { struct xe_config_device *dev = to_xe_config_device(item); @@ -187,6 +219,9 @@ static ssize_t survivability_mode_store(struct config_item *item, const char *pa return ret; guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + dev->config.survivability_mode = survivability_mode; return len; @@ -270,6 +305,9 @@ static ssize_t engines_allowed_store(struct config_item *item, const char *page, } guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + dev->config.engines_allowed = val; return len; @@ -293,6 +331,9 @@ static ssize_t enable_psmi_store(struct config_item *item, const char *page, siz return ret; guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + dev->config.enable_psmi = val; return len; From ce5059bf851b7286ec27797d786aab28c8a62e5e Mon Sep 17 00:00:00 2001 From: Stuart Summers Date: Tue, 26 Aug 2025 18:29:03 +0000 Subject: [PATCH 156/169] drm/xe: Move explicit CT lock in TLB invalidation sequence Currently the CT lock is used to cover TLB invalidation sequence number updates. In an effort to separate the GuC back end tracking of communication with the firmware from the front end TLB sequence number tracking, add a new lock here to specifically track those sequence number updates coming in from the user. Apart from the CT lock, we also have a pending lock to cover both pending fences and sequence numbers received from the back end. Those cover interrupt cases and so it makes not to overload those with sequence numbers coming in from new transactions. In that way, we'll employ a mutex here. v2: Actually add the correct lock rather than just dropping it... (Matt) Signed-off-by: Stuart Summers Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-2-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 19 +++++++++++++------ drivers/gpu/drm/xe/xe_gt_types.h | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c index 02f0bb92d6e0..75854b963d66 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c @@ -118,6 +118,9 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) */ int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt) { + struct xe_device *xe = gt_to_xe(gt); + int err; + gt->tlb_invalidation.seqno = 1; INIT_LIST_HEAD(>->tlb_invalidation.pending_fences); spin_lock_init(>->tlb_invalidation.pending_lock); @@ -125,6 +128,10 @@ int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt) INIT_DELAYED_WORK(>->tlb_invalidation.fence_tdr, xe_gt_tlb_fence_timeout); + err = drmm_mutex_init(&xe->drm, >->tlb_invalidation.seqno_lock); + if (err) + return err; + gt->tlb_invalidation.job_wq = drmm_alloc_ordered_workqueue(>_to_xe(gt)->drm, "gt-tbl-inval-job-wq", WQ_MEM_RECLAIM); @@ -158,7 +165,7 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) * appear. */ - mutex_lock(>->uc.guc.ct.lock); + mutex_lock(>->tlb_invalidation.seqno_lock); spin_lock_irq(>->tlb_invalidation.pending_lock); cancel_delayed_work(>->tlb_invalidation.fence_tdr); /* @@ -178,7 +185,7 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) >->tlb_invalidation.pending_fences, link) invalidation_fence_signal(gt_to_xe(gt), fence); spin_unlock_irq(>->tlb_invalidation.pending_lock); - mutex_unlock(>->uc.guc.ct.lock); + mutex_unlock(>->tlb_invalidation.seqno_lock); } static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) @@ -211,13 +218,13 @@ static int send_tlb_invalidation(struct xe_guc *guc, * need to be updated. */ - mutex_lock(&guc->ct.lock); + mutex_lock(>->tlb_invalidation.seqno_lock); seqno = gt->tlb_invalidation.seqno; fence->seqno = seqno; trace_xe_gt_tlb_invalidation_fence_send(xe, fence); action[1] = seqno; - ret = xe_guc_ct_send_locked(&guc->ct, action, len, - G2H_LEN_DW_TLB_INVALIDATE, 1); + ret = xe_guc_ct_send(&guc->ct, action, len, + G2H_LEN_DW_TLB_INVALIDATE, 1); if (!ret) { spin_lock_irq(>->tlb_invalidation.pending_lock); /* @@ -248,7 +255,7 @@ static int send_tlb_invalidation(struct xe_guc *guc, if (!gt->tlb_invalidation.seqno) gt->tlb_invalidation.seqno = 1; } - mutex_unlock(&guc->ct.lock); + mutex_unlock(>->tlb_invalidation.seqno_lock); xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); return ret; diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index ef0f2eecfa29..4dbc40fa6639 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -190,6 +190,8 @@ struct xe_gt { /** @tlb_invalidation.seqno: TLB invalidation seqno, protected by CT lock */ #define TLB_INVALIDATION_SEQNO_MAX 0x100000 int seqno; + /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ + struct mutex seqno_lock; /** * @tlb_invalidation.seqno_recv: last received TLB invalidation seqno, * protected by CT lock From 76186a253a4b9eb41c5a83224c14efdf30960a71 Mon Sep 17 00:00:00 2001 From: Stuart Summers Date: Tue, 26 Aug 2025 18:29:04 +0000 Subject: [PATCH 157/169] drm/xe: Cancel pending TLB inval workers on teardown Add a new _fini() routine on the GT TLB invalidation side to handle this worker cleanup on driver teardown. v2: Move the TLB teardown to the gt fini() routine called during gt_init rather than in gt_alloc. This way the GT structure stays alive for while we reset the TLB state. Signed-off-by: Stuart Summers Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-3-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_gt.c | 2 ++ drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 12 ++++++++++++ drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 1 + 3 files changed, 15 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index a3397f04abcc..178c4783bbda 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -603,6 +603,8 @@ static void xe_gt_fini(void *arg) struct xe_gt *gt = arg; int i; + xe_gt_tlb_invalidation_fini(gt); + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) xe_hw_fence_irq_finish(>->fence_irq[i]); diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c index 75854b963d66..db00c5adead9 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c @@ -188,6 +188,18 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) mutex_unlock(>->tlb_invalidation.seqno_lock); } +/** + * + * xe_gt_tlb_invalidation_fini - Clean up GT TLB invalidation state + * + * Cancel pending fence workers and clean up any additional + * GT TLB invalidation state. + */ +void xe_gt_tlb_invalidation_fini(struct xe_gt *gt) +{ + xe_gt_tlb_invalidation_reset(gt); +} + static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) { int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv); diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h index f7f0f2eaf4b5..3e4cff3922d6 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h @@ -16,6 +16,7 @@ struct xe_vm; struct xe_vma; int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt); +void xe_gt_tlb_invalidation_fini(struct xe_gt *gt); void xe_gt_tlb_invalidation_reset(struct xe_gt *gt); int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt); From c697ddcf27bd5c71b89539a9b79b7d078777b394 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:05 +0000 Subject: [PATCH 158/169] drm/xe: s/tlb_invalidation/tlb_inval tlb_invalidation is a bit verbose leading to ugly wraps in the code, shorten to tlb_inval. Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-4-stuart.summers@intel.com --- drivers/gpu/drm/xe/Makefile | 2 +- drivers/gpu/drm/xe/xe_device_types.h | 4 +- drivers/gpu/drm/xe/xe_exec_queue.c | 2 +- drivers/gpu/drm/xe/xe_ggtt.c | 4 +- drivers/gpu/drm/xe/xe_gt.c | 10 +- drivers/gpu/drm/xe/xe_gt_pagefault.c | 1 - ...t_tlb_invalidation.c => xe_gt_tlb_inval.c} | 262 +++++++++--------- drivers/gpu/drm/xe/xe_gt_tlb_inval.h | 41 +++ drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c | 18 +- ...dation_types.h => xe_gt_tlb_inval_types.h} | 14 +- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 41 --- drivers/gpu/drm/xe/xe_gt_types.h | 18 +- drivers/gpu/drm/xe/xe_guc_ct.c | 8 +- drivers/gpu/drm/xe/xe_lmtt.c | 12 +- drivers/gpu/drm/xe/xe_pci.c | 6 +- drivers/gpu/drm/xe/xe_pci_types.h | 2 +- drivers/gpu/drm/xe/xe_svm.c | 4 +- drivers/gpu/drm/xe/xe_trace.h | 24 +- drivers/gpu/drm/xe/xe_vm.c | 62 ++--- drivers/gpu/drm/xe/xe_vm.h | 4 +- drivers/gpu/drm/xe/xe_vm_madvise.c | 2 +- 21 files changed, 265 insertions(+), 276 deletions(-) rename drivers/gpu/drm/xe/{xe_gt_tlb_invalidation.c => xe_gt_tlb_inval.c} (61%) create mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_inval.h rename drivers/gpu/drm/xe/{xe_gt_tlb_invalidation_types.h => xe_gt_tlb_inval_types.h} (55%) delete mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 0abd8078cfb0..a1e4b3b1a1b6 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -61,7 +61,7 @@ xe-y += xe_bb.o \ xe_gt_pagefault.o \ xe_gt_sysfs.o \ xe_gt_throttle.o \ - xe_gt_tlb_invalidation.o \ + xe_gt_tlb_inval.o \ xe_gt_tlb_inval_job.o \ xe_gt_topology.o \ xe_guc.o \ diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 389996970564..646d04aec68c 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -290,8 +290,8 @@ struct xe_device { u8 has_mbx_power_limits:1; /** @info.has_pxp: Device has PXP support */ u8 has_pxp:1; - /** @info.has_range_tlb_invalidation: Has range based TLB invalidations */ - u8 has_range_tlb_invalidation:1; + /** @info.has_range_tlb_inval: Has range based TLB invalidations */ + u8 has_range_tlb_inval:1; /** @info.has_sriov: Supports SR-IOV */ u8 has_sriov:1; /** @info.has_usm: Device has unified shared memory support */ diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 2d10a53f701d..063c89d981e5 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -75,7 +75,7 @@ static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) if (!gt) continue; - wq = gt->tlb_invalidation.job_wq; + wq = gt->tlb_inval.job_wq; #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index e03222f5ac5a..c3e46c270117 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -23,7 +23,7 @@ #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_printk.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_map.h" #include "xe_mmio.h" #include "xe_pm.h" @@ -438,7 +438,7 @@ static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) if (!gt) return; - err = xe_gt_tlb_invalidation_ggtt(gt); + err = xe_gt_tlb_inval_ggtt(gt); xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err)); } diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 178c4783bbda..9a4639732bd7 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -37,7 +37,7 @@ #include "xe_gt_sriov_pf.h" #include "xe_gt_sriov_vf.h" #include "xe_gt_sysfs.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_gt_topology.h" #include "xe_guc_exec_queue_types.h" #include "xe_guc_pc.h" @@ -413,7 +413,7 @@ int xe_gt_init_early(struct xe_gt *gt) xe_force_wake_init_gt(gt, gt_to_fw(gt)); spin_lock_init(>->global_invl_lock); - err = xe_gt_tlb_invalidation_init_early(gt); + err = xe_gt_tlb_inval_init_early(gt); if (err) return err; @@ -603,7 +603,7 @@ static void xe_gt_fini(void *arg) struct xe_gt *gt = arg; int i; - xe_gt_tlb_invalidation_fini(gt); + xe_gt_tlb_inval_fini(gt); for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) xe_hw_fence_irq_finish(>->fence_irq[i]); @@ -852,7 +852,7 @@ static int gt_reset(struct xe_gt *gt) xe_uc_stop(>->uc); - xe_gt_tlb_invalidation_reset(gt); + xe_gt_tlb_inval_reset(gt); err = do_gt_reset(gt); if (err) @@ -1066,5 +1066,5 @@ void xe_gt_declare_wedged(struct xe_gt *gt) xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode); xe_uc_declare_wedged(>->uc); - xe_gt_tlb_invalidation_reset(gt); + xe_gt_tlb_inval_reset(gt); } diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index 4ea30fbce9bd..d02d22fb3659 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -16,7 +16,6 @@ #include "xe_gt.h" #include "xe_gt_printk.h" #include "xe_gt_stats.h" -#include "xe_gt_tlb_invalidation.h" #include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_migrate.h" diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_inval.c similarity index 61% rename from drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c rename to drivers/gpu/drm/xe/xe_gt_tlb_inval.c index db00c5adead9..1571fd917830 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval.c @@ -5,8 +5,6 @@ #include -#include "xe_gt_tlb_invalidation.h" - #include "abi/guc_actions_abi.h" #include "xe_device.h" #include "xe_force_wake.h" @@ -15,6 +13,7 @@ #include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_gt_stats.h" +#include "xe_gt_tlb_inval.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_sriov.h" @@ -39,7 +38,7 @@ static long tlb_timeout_jiffies(struct xe_gt *gt) return hw_tlb_timeout + 2 * delay; } -static void xe_gt_tlb_invalidation_fence_fini(struct xe_gt_tlb_invalidation_fence *fence) +static void xe_gt_tlb_inval_fence_fini(struct xe_gt_tlb_inval_fence *fence) { if (WARN_ON_ONCE(!fence->gt)) return; @@ -49,66 +48,66 @@ static void xe_gt_tlb_invalidation_fence_fini(struct xe_gt_tlb_invalidation_fenc } static void -__invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence) +__inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence) { bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags); - trace_xe_gt_tlb_invalidation_fence_signal(xe, fence); - xe_gt_tlb_invalidation_fence_fini(fence); + trace_xe_gt_tlb_inval_fence_signal(xe, fence); + xe_gt_tlb_inval_fence_fini(fence); dma_fence_signal(&fence->base); if (!stack) dma_fence_put(&fence->base); } static void -invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence) +inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence) { list_del(&fence->link); - __invalidation_fence_signal(xe, fence); + __inval_fence_signal(xe, fence); } -void xe_gt_tlb_invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence) +void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence) { if (WARN_ON_ONCE(!fence->gt)) return; - __invalidation_fence_signal(gt_to_xe(fence->gt), fence); + __inval_fence_signal(gt_to_xe(fence->gt), fence); } static void xe_gt_tlb_fence_timeout(struct work_struct *work) { struct xe_gt *gt = container_of(work, struct xe_gt, - tlb_invalidation.fence_tdr.work); + tlb_inval.fence_tdr.work); struct xe_device *xe = gt_to_xe(gt); - struct xe_gt_tlb_invalidation_fence *fence, *next; + struct xe_gt_tlb_inval_fence *fence, *next; LNL_FLUSH_WORK(>->uc.guc.ct.g2h_worker); - spin_lock_irq(>->tlb_invalidation.pending_lock); + spin_lock_irq(>->tlb_inval.pending_lock); list_for_each_entry_safe(fence, next, - >->tlb_invalidation.pending_fences, link) { + >->tlb_inval.pending_fences, link) { s64 since_inval_ms = ktime_ms_delta(ktime_get(), - fence->invalidation_time); + fence->inval_time); if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt)) break; - trace_xe_gt_tlb_invalidation_fence_timeout(xe, fence); + trace_xe_gt_tlb_inval_fence_timeout(xe, fence); xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d", - fence->seqno, gt->tlb_invalidation.seqno_recv); + fence->seqno, gt->tlb_inval.seqno_recv); fence->base.error = -ETIME; - invalidation_fence_signal(xe, fence); + inval_fence_signal(xe, fence); } - if (!list_empty(>->tlb_invalidation.pending_fences)) + if (!list_empty(>->tlb_inval.pending_fences)) queue_delayed_work(system_wq, - >->tlb_invalidation.fence_tdr, + >->tlb_inval.fence_tdr, tlb_timeout_jiffies(gt)); - spin_unlock_irq(>->tlb_invalidation.pending_lock); + spin_unlock_irq(>->tlb_inval.pending_lock); } /** - * xe_gt_tlb_invalidation_init_early - Initialize GT TLB invalidation state + * xe_gt_tlb_inval_init_early - Initialize GT TLB invalidation state * @gt: GT structure * * Initialize GT TLB invalidation state, purely software initialization, should @@ -116,40 +115,40 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) * * Return: 0 on success, negative error code on error. */ -int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt) +int xe_gt_tlb_inval_init_early(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); int err; - gt->tlb_invalidation.seqno = 1; - INIT_LIST_HEAD(>->tlb_invalidation.pending_fences); - spin_lock_init(>->tlb_invalidation.pending_lock); - spin_lock_init(>->tlb_invalidation.lock); - INIT_DELAYED_WORK(>->tlb_invalidation.fence_tdr, + gt->tlb_inval.seqno = 1; + INIT_LIST_HEAD(>->tlb_inval.pending_fences); + spin_lock_init(>->tlb_inval.pending_lock); + spin_lock_init(>->tlb_inval.lock); + INIT_DELAYED_WORK(>->tlb_inval.fence_tdr, xe_gt_tlb_fence_timeout); - err = drmm_mutex_init(&xe->drm, >->tlb_invalidation.seqno_lock); + err = drmm_mutex_init(&xe->drm, >->tlb_inval.seqno_lock); if (err) return err; - gt->tlb_invalidation.job_wq = + gt->tlb_inval.job_wq = drmm_alloc_ordered_workqueue(>_to_xe(gt)->drm, "gt-tbl-inval-job-wq", WQ_MEM_RECLAIM); - if (IS_ERR(gt->tlb_invalidation.job_wq)) - return PTR_ERR(gt->tlb_invalidation.job_wq); + if (IS_ERR(gt->tlb_inval.job_wq)) + return PTR_ERR(gt->tlb_inval.job_wq); return 0; } /** - * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset + * xe_gt_tlb_inval_reset - Initialize GT TLB invalidation reset * @gt: GT structure * * Signal any pending invalidation fences, should be called during a GT reset */ -void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) +void xe_gt_tlb_inval_reset(struct xe_gt *gt) { - struct xe_gt_tlb_invalidation_fence *fence, *next; + struct xe_gt_tlb_inval_fence *fence, *next; int pending_seqno; /* @@ -165,9 +164,9 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) * appear. */ - mutex_lock(>->tlb_invalidation.seqno_lock); - spin_lock_irq(>->tlb_invalidation.pending_lock); - cancel_delayed_work(>->tlb_invalidation.fence_tdr); + mutex_lock(>->tlb_inval.seqno_lock); + spin_lock_irq(>->tlb_inval.pending_lock); + cancel_delayed_work(>->tlb_inval.fence_tdr); /* * We might have various kworkers waiting for TLB flushes to complete * which are not tracked with an explicit TLB fence, however at this @@ -175,34 +174,34 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) * make sure we signal them here under the assumption that we have * completed a full GT reset. */ - if (gt->tlb_invalidation.seqno == 1) + if (gt->tlb_inval.seqno == 1) pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1; else - pending_seqno = gt->tlb_invalidation.seqno - 1; - WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno); + pending_seqno = gt->tlb_inval.seqno - 1; + WRITE_ONCE(gt->tlb_inval.seqno_recv, pending_seqno); list_for_each_entry_safe(fence, next, - >->tlb_invalidation.pending_fences, link) - invalidation_fence_signal(gt_to_xe(gt), fence); - spin_unlock_irq(>->tlb_invalidation.pending_lock); - mutex_unlock(>->tlb_invalidation.seqno_lock); + >->tlb_inval.pending_fences, link) + inval_fence_signal(gt_to_xe(gt), fence); + spin_unlock_irq(>->tlb_inval.pending_lock); + mutex_unlock(>->tlb_inval.seqno_lock); } /** * - * xe_gt_tlb_invalidation_fini - Clean up GT TLB invalidation state + * xe_gt_tlb_inval_fini - Clean up GT TLB invalidation state * * Cancel pending fence workers and clean up any additional * GT TLB invalidation state. */ -void xe_gt_tlb_invalidation_fini(struct xe_gt *gt) +void xe_gt_tlb_inval_fini(struct xe_gt *gt) { - xe_gt_tlb_invalidation_reset(gt); + xe_gt_tlb_inval_reset(gt); } -static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) +static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) { - int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv); + int seqno_recv = READ_ONCE(gt->tlb_inval.seqno_recv); if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2)) return false; @@ -213,9 +212,9 @@ static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) return seqno_recv >= seqno; } -static int send_tlb_invalidation(struct xe_guc *guc, - struct xe_gt_tlb_invalidation_fence *fence, - u32 *action, int len) +static int send_tlb_inval(struct xe_guc *guc, + struct xe_gt_tlb_inval_fence *fence, + u32 *action, int len) { struct xe_gt *gt = guc_to_gt(guc); struct xe_device *xe = gt_to_xe(gt); @@ -230,44 +229,44 @@ static int send_tlb_invalidation(struct xe_guc *guc, * need to be updated. */ - mutex_lock(>->tlb_invalidation.seqno_lock); - seqno = gt->tlb_invalidation.seqno; + mutex_lock(>->tlb_inval.seqno_lock); + seqno = gt->tlb_inval.seqno; fence->seqno = seqno; - trace_xe_gt_tlb_invalidation_fence_send(xe, fence); + trace_xe_gt_tlb_inval_fence_send(xe, fence); action[1] = seqno; ret = xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_TLB_INVALIDATE, 1); if (!ret) { - spin_lock_irq(>->tlb_invalidation.pending_lock); + spin_lock_irq(>->tlb_inval.pending_lock); /* * We haven't actually published the TLB fence as per * pending_fences, but in theory our seqno could have already * been written as we acquired the pending_lock. In such a case * we can just go ahead and signal the fence here. */ - if (tlb_invalidation_seqno_past(gt, seqno)) { - __invalidation_fence_signal(xe, fence); + if (tlb_inval_seqno_past(gt, seqno)) { + __inval_fence_signal(xe, fence); } else { - fence->invalidation_time = ktime_get(); + fence->inval_time = ktime_get(); list_add_tail(&fence->link, - >->tlb_invalidation.pending_fences); + >->tlb_inval.pending_fences); - if (list_is_singular(>->tlb_invalidation.pending_fences)) + if (list_is_singular(>->tlb_inval.pending_fences)) queue_delayed_work(system_wq, - >->tlb_invalidation.fence_tdr, + >->tlb_inval.fence_tdr, tlb_timeout_jiffies(gt)); } - spin_unlock_irq(>->tlb_invalidation.pending_lock); + spin_unlock_irq(>->tlb_inval.pending_lock); } else { - __invalidation_fence_signal(xe, fence); + __inval_fence_signal(xe, fence); } if (!ret) { - gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) % + gt->tlb_inval.seqno = (gt->tlb_inval.seqno + 1) % TLB_INVALIDATION_SEQNO_MAX; - if (!gt->tlb_invalidation.seqno) - gt->tlb_invalidation.seqno = 1; + if (!gt->tlb_inval.seqno) + gt->tlb_inval.seqno = 1; } - mutex_unlock(>->tlb_invalidation.seqno_lock); + mutex_unlock(>->tlb_inval.seqno_lock); xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); return ret; @@ -278,7 +277,7 @@ static int send_tlb_invalidation(struct xe_guc *guc, XE_GUC_TLB_INVAL_FLUSH_CACHE) /** - * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC + * xe_gt_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC * @gt: GT structure * @fence: invalidation fence which will be signal on TLB invalidation * completion @@ -288,18 +287,17 @@ static int send_tlb_invalidation(struct xe_guc *guc, * * Return: 0 on success, negative error code on error */ -static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence) +static int xe_gt_tlb_inval_guc(struct xe_gt *gt, + struct xe_gt_tlb_inval_fence *fence) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION, - 0, /* seqno, replaced in send_tlb_invalidation */ + 0, /* seqno, replaced in send_tlb_inval */ MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), }; int ret; - ret = send_tlb_invalidation(>->uc.guc, fence, action, - ARRAY_SIZE(action)); + ret = send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); /* * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches * should be nuked on a GT reset so this error can be ignored. @@ -311,7 +309,7 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt, } /** - * xe_gt_tlb_invalidation_ggtt - Issue a TLB invalidation on this GT for the GGTT + * xe_gt_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT * @gt: GT structure * * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is @@ -319,22 +317,22 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt, * * Return: 0 on success, negative error code on error */ -int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt) +int xe_gt_tlb_inval_ggtt(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); unsigned int fw_ref; if (xe_guc_ct_enabled(>->uc.guc.ct) && gt->uc.guc.submission_state.enabled) { - struct xe_gt_tlb_invalidation_fence fence; + struct xe_gt_tlb_inval_fence fence; int ret; - xe_gt_tlb_invalidation_fence_init(gt, &fence, true); - ret = xe_gt_tlb_invalidation_guc(gt, &fence); + xe_gt_tlb_inval_fence_init(gt, &fence, true); + ret = xe_gt_tlb_inval_guc(gt, &fence); if (ret) return ret; - xe_gt_tlb_invalidation_fence_wait(&fence); + xe_gt_tlb_inval_fence_wait(&fence); } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { struct xe_mmio *mmio = >->mmio; @@ -357,34 +355,34 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt) return 0; } -static int send_tlb_invalidation_all(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence) +static int send_tlb_inval_all(struct xe_gt *gt, + struct xe_gt_tlb_inval_fence *fence) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION_ALL, - 0, /* seqno, replaced in send_tlb_invalidation */ + 0, /* seqno, replaced in send_tlb_inval */ MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), }; - return send_tlb_invalidation(>->uc.guc, fence, action, ARRAY_SIZE(action)); + return send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); } /** * xe_gt_tlb_invalidation_all - Invalidate all TLBs across PF and all VFs. * @gt: the &xe_gt structure - * @fence: the &xe_gt_tlb_invalidation_fence to be signaled on completion + * @fence: the &xe_gt_tlb_inval_fence to be signaled on completion * * Send a request to invalidate all TLBs across PF and all VFs. * * Return: 0 on success, negative error code on error */ -int xe_gt_tlb_invalidation_all(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence) +int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence) { int err; xe_gt_assert(gt, gt == fence->gt); - err = send_tlb_invalidation_all(gt, fence); + err = send_tlb_inval_all(gt, fence); if (err) xe_gt_err(gt, "TLB invalidation request failed (%pe)", ERR_PTR(err)); @@ -399,8 +397,7 @@ int xe_gt_tlb_invalidation_all(struct xe_gt *gt, struct xe_gt_tlb_invalidation_f #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) /** - * xe_gt_tlb_invalidation_range - Issue a TLB invalidation on this GT for an - * address range + * xe_gt_tlb_inval_range - Issue a TLB invalidation on this GT for an address range * * @gt: GT structure * @fence: invalidation fence which will be signal on TLB invalidation @@ -415,9 +412,8 @@ int xe_gt_tlb_invalidation_all(struct xe_gt *gt, struct xe_gt_tlb_invalidation_f * * Return: Negative error code on error, 0 on success */ -int xe_gt_tlb_invalidation_range(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence, - u64 start, u64 end, u32 asid) +int xe_gt_tlb_inval_range(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence, + u64 start, u64 end, u32 asid) { struct xe_device *xe = gt_to_xe(gt); #define MAX_TLB_INVALIDATION_LEN 7 @@ -429,13 +425,13 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, /* Execlists not supported */ if (gt_to_xe(gt)->info.force_execlist) { - __invalidation_fence_signal(xe, fence); + __inval_fence_signal(xe, fence); return 0; } action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; - action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */ - if (!xe->info.has_range_tlb_invalidation || + action[len++] = 0; /* seqno, replaced in send_tlb_inval */ + if (!xe->info.has_range_tlb_inval || length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); } else { @@ -484,33 +480,33 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); - return send_tlb_invalidation(>->uc.guc, fence, action, len); + return send_tlb_inval(>->uc.guc, fence, action, len); } /** - * xe_gt_tlb_invalidation_vm - Issue a TLB invalidation on this GT for a VM + * xe_gt_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM * @gt: graphics tile * @vm: VM to invalidate * * Invalidate entire VM's address space */ -void xe_gt_tlb_invalidation_vm(struct xe_gt *gt, struct xe_vm *vm) +void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm) { - struct xe_gt_tlb_invalidation_fence fence; + struct xe_gt_tlb_inval_fence fence; u64 range = 1ull << vm->xe->info.va_bits; int ret; - xe_gt_tlb_invalidation_fence_init(gt, &fence, true); + xe_gt_tlb_inval_fence_init(gt, &fence, true); - ret = xe_gt_tlb_invalidation_range(gt, &fence, 0, range, vm->usm.asid); + ret = xe_gt_tlb_inval_range(gt, &fence, 0, range, vm->usm.asid); if (ret < 0) return; - xe_gt_tlb_invalidation_fence_wait(&fence); + xe_gt_tlb_inval_fence_wait(&fence); } /** - * xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler + * xe_guc_tlb_inval_done_handler - TLB invalidation done handler * @guc: guc * @msg: message indicating TLB invalidation done * @len: length of message @@ -521,11 +517,11 @@ void xe_gt_tlb_invalidation_vm(struct xe_gt *gt, struct xe_vm *vm) * * Return: 0 on success, -EPROTO for malformed messages. */ -int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_gt *gt = guc_to_gt(guc); struct xe_device *xe = gt_to_xe(gt); - struct xe_gt_tlb_invalidation_fence *fence, *next; + struct xe_gt_tlb_inval_fence *fence, *next; unsigned long flags; if (unlikely(len != 1)) @@ -546,74 +542,74 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len) * officially process the CT message like if racing against * process_g2h_msg(). */ - spin_lock_irqsave(>->tlb_invalidation.pending_lock, flags); - if (tlb_invalidation_seqno_past(gt, msg[0])) { - spin_unlock_irqrestore(>->tlb_invalidation.pending_lock, flags); + spin_lock_irqsave(>->tlb_inval.pending_lock, flags); + if (tlb_inval_seqno_past(gt, msg[0])) { + spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); return 0; } - WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]); + WRITE_ONCE(gt->tlb_inval.seqno_recv, msg[0]); list_for_each_entry_safe(fence, next, - >->tlb_invalidation.pending_fences, link) { - trace_xe_gt_tlb_invalidation_fence_recv(xe, fence); + >->tlb_inval.pending_fences, link) { + trace_xe_gt_tlb_inval_fence_recv(xe, fence); - if (!tlb_invalidation_seqno_past(gt, fence->seqno)) + if (!tlb_inval_seqno_past(gt, fence->seqno)) break; - invalidation_fence_signal(xe, fence); + inval_fence_signal(xe, fence); } - if (!list_empty(>->tlb_invalidation.pending_fences)) + if (!list_empty(>->tlb_inval.pending_fences)) mod_delayed_work(system_wq, - >->tlb_invalidation.fence_tdr, + >->tlb_inval.fence_tdr, tlb_timeout_jiffies(gt)); else - cancel_delayed_work(>->tlb_invalidation.fence_tdr); + cancel_delayed_work(>->tlb_inval.fence_tdr); - spin_unlock_irqrestore(>->tlb_invalidation.pending_lock, flags); + spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); return 0; } static const char * -invalidation_fence_get_driver_name(struct dma_fence *dma_fence) +inval_fence_get_driver_name(struct dma_fence *dma_fence) { return "xe"; } static const char * -invalidation_fence_get_timeline_name(struct dma_fence *dma_fence) +inval_fence_get_timeline_name(struct dma_fence *dma_fence) { - return "invalidation_fence"; + return "inval_fence"; } -static const struct dma_fence_ops invalidation_fence_ops = { - .get_driver_name = invalidation_fence_get_driver_name, - .get_timeline_name = invalidation_fence_get_timeline_name, +static const struct dma_fence_ops inval_fence_ops = { + .get_driver_name = inval_fence_get_driver_name, + .get_timeline_name = inval_fence_get_timeline_name, }; /** - * xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence + * xe_gt_tlb_inval_fence_init - Initialize TLB invalidation fence * @gt: GT * @fence: TLB invalidation fence to initialize * @stack: fence is stack variable * - * Initialize TLB invalidation fence for use. xe_gt_tlb_invalidation_fence_fini + * Initialize TLB invalidation fence for use. xe_gt_tlb_inval_fence_fini * will be automatically called when fence is signalled (all fences must signal), * even on error. */ -void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence, - bool stack) +void xe_gt_tlb_inval_fence_init(struct xe_gt *gt, + struct xe_gt_tlb_inval_fence *fence, + bool stack) { xe_pm_runtime_get_noresume(gt_to_xe(gt)); - spin_lock_irq(>->tlb_invalidation.lock); - dma_fence_init(&fence->base, &invalidation_fence_ops, - >->tlb_invalidation.lock, + spin_lock_irq(>->tlb_inval.lock); + dma_fence_init(&fence->base, &inval_fence_ops, + >->tlb_inval.lock, dma_fence_context_alloc(1), 1); - spin_unlock_irq(>->tlb_invalidation.lock); + spin_unlock_irq(>->tlb_inval.lock); INIT_LIST_HEAD(&fence->link); if (stack) set_bit(FENCE_STACK_BIT, &fence->base.flags); diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval.h new file mode 100644 index 000000000000..b1258ac4e8fb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GT_TLB_INVAL_H_ +#define _XE_GT_TLB_INVAL_H_ + +#include + +#include "xe_gt_tlb_inval_types.h" + +struct xe_gt; +struct xe_guc; +struct xe_vm; +struct xe_vma; + +int xe_gt_tlb_inval_init_early(struct xe_gt *gt); +void xe_gt_tlb_inval_fini(struct xe_gt *gt); + +void xe_gt_tlb_inval_reset(struct xe_gt *gt); +int xe_gt_tlb_inval_ggtt(struct xe_gt *gt); +void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm); +int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence); +int xe_gt_tlb_inval_range(struct xe_gt *gt, + struct xe_gt_tlb_inval_fence *fence, + u64 start, u64 end, u32 asid); +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); + +void xe_gt_tlb_inval_fence_init(struct xe_gt *gt, + struct xe_gt_tlb_inval_fence *fence, + bool stack); +void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence); + +static inline void +xe_gt_tlb_inval_fence_wait(struct xe_gt_tlb_inval_fence *fence) +{ + dma_fence_wait(&fence->base, false); +} + +#endif /* _XE_GT_TLB_INVAL_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c index e9255be26467..41e0ea92ea5a 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c @@ -7,7 +7,7 @@ #include "xe_dep_scheduler.h" #include "xe_exec_queue.h" #include "xe_gt.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_gt_tlb_inval_job.h" #include "xe_migrate.h" #include "xe_pm.h" @@ -41,11 +41,11 @@ static struct dma_fence *xe_gt_tlb_inval_job_run(struct xe_dep_job *dep_job) { struct xe_gt_tlb_inval_job *job = container_of(dep_job, typeof(*job), dep); - struct xe_gt_tlb_invalidation_fence *ifence = + struct xe_gt_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); - xe_gt_tlb_invalidation_range(job->gt, ifence, job->start, - job->end, job->asid); + xe_gt_tlb_inval_range(job->gt, ifence, job->start, + job->end, job->asid); return job->fence; } @@ -93,7 +93,7 @@ struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, q->tlb_inval[xe_gt_tlb_inval_context(gt)].dep_scheduler; struct drm_sched_entity *entity = xe_dep_scheduler_entity(dep_scheduler); - struct xe_gt_tlb_invalidation_fence *ifence; + struct xe_gt_tlb_inval_fence *ifence; int err; job = kmalloc(sizeof(*job), GFP_KERNEL); @@ -140,7 +140,7 @@ static void xe_gt_tlb_inval_job_destroy(struct kref *ref) { struct xe_gt_tlb_inval_job *job = container_of(ref, typeof(*job), refcount); - struct xe_gt_tlb_invalidation_fence *ifence = + struct xe_gt_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); struct xe_device *xe = gt_to_xe(job->gt); struct xe_exec_queue *q = job->q; @@ -148,7 +148,7 @@ static void xe_gt_tlb_inval_job_destroy(struct kref *ref) if (!job->fence_armed) kfree(ifence); else - /* Ref from xe_gt_tlb_invalidation_fence_init */ + /* Ref from xe_gt_tlb_inval_fence_init */ dma_fence_put(job->fence); drm_sched_job_cleanup(&job->dep.drm); @@ -194,7 +194,7 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, struct xe_migrate *m, struct dma_fence *fence) { - struct xe_gt_tlb_invalidation_fence *ifence = + struct xe_gt_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); if (!dma_fence_is_signaled(fence)) { @@ -226,7 +226,7 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, xe_migrate_job_lock(m, job->q); /* Creation ref pairs with put in xe_gt_tlb_inval_job_destroy */ - xe_gt_tlb_invalidation_fence_init(job->gt, ifence, false); + xe_gt_tlb_inval_fence_init(job->gt, ifence, false); dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */ drm_sched_job_arm(&job->dep.drm); diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h similarity index 55% rename from drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h rename to drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h index de6e825e0851..919430359103 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h @@ -3,20 +3,20 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_GT_TLB_INVALIDATION_TYPES_H_ -#define _XE_GT_TLB_INVALIDATION_TYPES_H_ +#ifndef _XE_GT_TLB_INVAL_TYPES_H_ +#define _XE_GT_TLB_INVAL_TYPES_H_ #include struct xe_gt; /** - * struct xe_gt_tlb_invalidation_fence - XE GT TLB invalidation fence + * struct xe_gt_tlb_inval_fence - XE GT TLB invalidation fence * - * Optionally passed to xe_gt_tlb_invalidation and will be signaled upon TLB + * Optionally passed to xe_gt_tlb_inval and will be signaled upon TLB * invalidation completion. */ -struct xe_gt_tlb_invalidation_fence { +struct xe_gt_tlb_inval_fence { /** @base: dma fence base */ struct dma_fence base; /** @gt: GT which fence belong to */ @@ -25,8 +25,8 @@ struct xe_gt_tlb_invalidation_fence { struct list_head link; /** @seqno: seqno of TLB invalidation to signal fence one */ int seqno; - /** @invalidation_time: time of TLB invalidation */ - ktime_t invalidation_time; + /** @inval_time: time of TLB invalidation */ + ktime_t inval_time; }; #endif diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h deleted file mode 100644 index 3e4cff3922d6..000000000000 --- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef _XE_GT_TLB_INVALIDATION_H_ -#define _XE_GT_TLB_INVALIDATION_H_ - -#include - -#include "xe_gt_tlb_invalidation_types.h" - -struct xe_gt; -struct xe_guc; -struct xe_vm; -struct xe_vma; - -int xe_gt_tlb_invalidation_init_early(struct xe_gt *gt); -void xe_gt_tlb_invalidation_fini(struct xe_gt *gt); - -void xe_gt_tlb_invalidation_reset(struct xe_gt *gt); -int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt); -void xe_gt_tlb_invalidation_vm(struct xe_gt *gt, struct xe_vm *vm); -int xe_gt_tlb_invalidation_all(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence); -int xe_gt_tlb_invalidation_range(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence, - u64 start, u64 end, u32 asid); -int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len); - -void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt, - struct xe_gt_tlb_invalidation_fence *fence, - bool stack); -void xe_gt_tlb_invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence); - -static inline void -xe_gt_tlb_invalidation_fence_wait(struct xe_gt_tlb_invalidation_fence *fence) -{ - dma_fence_wait(&fence->base, false); -} - -#endif /* _XE_GT_TLB_INVALIDATION_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 4dbc40fa6639..85cfcc49472b 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -185,38 +185,38 @@ struct xe_gt { struct work_struct worker; } reset; - /** @tlb_invalidation: TLB invalidation state */ + /** @tlb_inval: TLB invalidation state */ struct { - /** @tlb_invalidation.seqno: TLB invalidation seqno, protected by CT lock */ + /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ #define TLB_INVALIDATION_SEQNO_MAX 0x100000 int seqno; /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ struct mutex seqno_lock; /** - * @tlb_invalidation.seqno_recv: last received TLB invalidation seqno, + * @tlb_inval.seqno_recv: last received TLB invalidation seqno, * protected by CT lock */ int seqno_recv; /** - * @tlb_invalidation.pending_fences: list of pending fences waiting TLB + * @tlb_inval.pending_fences: list of pending fences waiting TLB * invaliations, protected by CT lock */ struct list_head pending_fences; /** - * @tlb_invalidation.pending_lock: protects @tlb_invalidation.pending_fences - * and updating @tlb_invalidation.seqno_recv. + * @tlb_inval.pending_lock: protects @tlb_inval.pending_fences + * and updating @tlb_inval.seqno_recv. */ spinlock_t pending_lock; /** - * @tlb_invalidation.fence_tdr: schedules a delayed call to + * @tlb_inval.fence_tdr: schedules a delayed call to * xe_gt_tlb_fence_timeout after the timeut interval is over. */ struct delayed_work fence_tdr; /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */ struct workqueue_struct *job_wq; - /** @tlb_invalidation.lock: protects TLB invalidation fences */ + /** @tlb_inval.lock: protects TLB invalidation fences */ spinlock_t lock; - } tlb_invalidation; + } tlb_inval; /** * @ccs_mode: Number of compute engines enabled. diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 3f4e6a46ff16..9131d121d941 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -26,7 +26,7 @@ #include "xe_gt_sriov_pf_control.h" #include "xe_gt_sriov_pf_monitor.h" #include "xe_gt_sriov_printk.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_guc.h" #include "xe_guc_log.h" #include "xe_guc_relay.h" @@ -1416,8 +1416,7 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_pagefault_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_TLB_INVALIDATION_DONE: - ret = xe_guc_tlb_invalidation_done_handler(guc, payload, - adj_len); + ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: ret = xe_guc_access_counter_notify_handler(guc, payload, @@ -1618,8 +1617,7 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) break; case XE_GUC_ACTION_TLB_INVALIDATION_DONE: __g2h_release_space(ct, len); - ret = xe_guc_tlb_invalidation_done_handler(guc, payload, - adj_len); + ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); break; default: xe_gt_warn(gt, "NOT_POSSIBLE"); diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c index a78c9d474a6e..e5aba03ff8ac 100644 --- a/drivers/gpu/drm/xe/xe_lmtt.c +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -11,7 +11,7 @@ #include "xe_assert.h" #include "xe_bo.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_lmtt.h" #include "xe_map.h" #include "xe_mmio.h" @@ -228,8 +228,8 @@ void xe_lmtt_init_hw(struct xe_lmtt *lmtt) static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) { - struct xe_gt_tlb_invalidation_fence fences[XE_MAX_GT_PER_TILE]; - struct xe_gt_tlb_invalidation_fence *fence = fences; + struct xe_gt_tlb_inval_fence fences[XE_MAX_GT_PER_TILE]; + struct xe_gt_tlb_inval_fence *fence = fences; struct xe_tile *tile = lmtt_to_tile(lmtt); struct xe_gt *gt; int result = 0; @@ -237,8 +237,8 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) u8 id; for_each_gt_on_tile(gt, tile, id) { - xe_gt_tlb_invalidation_fence_init(gt, fence, true); - err = xe_gt_tlb_invalidation_all(gt, fence); + xe_gt_tlb_inval_fence_init(gt, fence, true); + err = xe_gt_tlb_inval_all(gt, fence); result = result ?: err; fence++; } @@ -252,7 +252,7 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) */ fence = fences; for_each_gt_on_tile(gt, tile, id) - xe_gt_tlb_invalidation_fence_wait(fence++); + xe_gt_tlb_inval_fence_wait(fence++); return result; } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 15a863491cdd..046d330bad34 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -57,7 +57,7 @@ static const struct xe_graphics_desc graphics_xelp = { }; #define XE_HP_FEATURES \ - .has_range_tlb_invalidation = true, \ + .has_range_tlb_inval = true, \ .va_bits = 48, \ .vm_max_level = 3 @@ -105,7 +105,7 @@ static const struct xe_graphics_desc graphics_xelpg = { .has_asid = 1, \ .has_atomic_enable_pte_bit = 1, \ .has_flat_ccs = 1, \ - .has_range_tlb_invalidation = 1, \ + .has_range_tlb_inval = 1, \ .has_usm = 1, \ .has_64bit_timestamp = 1, \ .va_bits = 48, \ @@ -713,7 +713,7 @@ static int xe_info_init(struct xe_device *xe, /* Runtime detection may change this later */ xe->info.has_flat_ccs = graphics_desc->has_flat_ccs; - xe->info.has_range_tlb_invalidation = graphics_desc->has_range_tlb_invalidation; + xe->info.has_range_tlb_inval = graphics_desc->has_range_tlb_inval; xe->info.has_usm = graphics_desc->has_usm; xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 4de6f69ed975..b63002fc0f67 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -60,7 +60,7 @@ struct xe_graphics_desc { u8 has_atomic_enable_pte_bit:1; u8 has_flat_ccs:1; u8 has_indirect_ring_state:1; - u8 has_range_tlb_invalidation:1; + u8 has_range_tlb_inval:1; u8 has_usm:1; u8 has_64bit_timestamp:1; }; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 0596039ef0a1..babc64485b52 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -7,7 +7,7 @@ #include "xe_bo.h" #include "xe_gt_stats.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_migrate.h" #include "xe_module.h" #include "xe_pm.h" @@ -225,7 +225,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, xe_device_wmb(xe); - err = xe_vm_range_tilemask_tlb_invalidation(vm, adj_start, adj_end, tile_mask); + err = xe_vm_range_tilemask_tlb_inval(vm, adj_start, adj_end, tile_mask); WARN_ON_ONCE(err); range_notifier_event_end: diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index 21486a6f693a..36538f50d06f 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -14,7 +14,7 @@ #include "xe_exec_queue_types.h" #include "xe_gpu_scheduler_types.h" -#include "xe_gt_tlb_invalidation_types.h" +#include "xe_gt_tlb_inval_types.h" #include "xe_gt_types.h" #include "xe_guc_exec_queue_types.h" #include "xe_sched_job.h" @@ -25,13 +25,13 @@ #define __dev_name_gt(gt) __dev_name_xe(gt_to_xe((gt))) #define __dev_name_eq(q) __dev_name_gt((q)->gt) -DECLARE_EVENT_CLASS(xe_gt_tlb_invalidation_fence, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), +DECLARE_EVENT_CLASS(xe_gt_tlb_inval_fence, + TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), TP_ARGS(xe, fence), TP_STRUCT__entry( __string(dev, __dev_name_xe(xe)) - __field(struct xe_gt_tlb_invalidation_fence *, fence) + __field(struct xe_gt_tlb_inval_fence *, fence) __field(int, seqno) ), @@ -45,23 +45,23 @@ DECLARE_EVENT_CLASS(xe_gt_tlb_invalidation_fence, __get_str(dev), __entry->fence, __entry->seqno) ); -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_send, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), +DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_send, + TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_recv, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), +DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_recv, + TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_signal, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), +DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_signal, + TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_invalidation_fence, xe_gt_tlb_invalidation_fence_timeout, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_invalidation_fence *fence), +DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_timeout, + TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index e913efa2057b..b758e9a6837a 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -28,7 +28,7 @@ #include "xe_drm_client.h" #include "xe_exec_queue.h" #include "xe_gt_pagefault.h" -#include "xe_gt_tlb_invalidation.h" +#include "xe_gt_tlb_inval.h" #include "xe_migrate.h" #include "xe_pat.h" #include "xe_pm.h" @@ -1898,7 +1898,7 @@ static void xe_vm_close(struct xe_vm *vm) xe_pt_clear(xe, vm->pt_root[id]); for_each_gt(gt, xe, id) - xe_gt_tlb_invalidation_vm(gt, vm); + xe_gt_tlb_inval_vm(gt, vm); } } @@ -4032,7 +4032,7 @@ void xe_vm_unlock(struct xe_vm *vm) } /** - * xe_vm_range_tilemask_tlb_invalidation - Issue a TLB invalidation on this tilemask for an + * xe_vm_range_tilemask_tlb_inval - Issue a TLB invalidation on this tilemask for an * address range * @vm: The VM * @start: start address @@ -4043,10 +4043,11 @@ void xe_vm_unlock(struct xe_vm *vm) * * Returns 0 for success, negative error code otherwise. */ -int xe_vm_range_tilemask_tlb_invalidation(struct xe_vm *vm, u64 start, - u64 end, u8 tile_mask) +int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, + u64 end, u8 tile_mask) { - struct xe_gt_tlb_invalidation_fence fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; + struct xe_gt_tlb_inval_fence + fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; struct xe_tile *tile; u32 fence_id = 0; u8 id; @@ -4056,39 +4057,34 @@ int xe_vm_range_tilemask_tlb_invalidation(struct xe_vm *vm, u64 start, return 0; for_each_tile(tile, vm->xe, id) { - if (tile_mask & BIT(id)) { - xe_gt_tlb_invalidation_fence_init(tile->primary_gt, - &fence[fence_id], true); + if (!(tile_mask & BIT(id))) + continue; - err = xe_gt_tlb_invalidation_range(tile->primary_gt, - &fence[fence_id], - start, - end, - vm->usm.asid); - if (err) - goto wait; - ++fence_id; + xe_gt_tlb_inval_fence_init(tile->primary_gt, + &fence[fence_id], true); - if (!tile->media_gt) - continue; + err = xe_gt_tlb_inval_range(tile->primary_gt, &fence[fence_id], + start, end, vm->usm.asid); + if (err) + goto wait; + ++fence_id; - xe_gt_tlb_invalidation_fence_init(tile->media_gt, - &fence[fence_id], true); + if (!tile->media_gt) + continue; - err = xe_gt_tlb_invalidation_range(tile->media_gt, - &fence[fence_id], - start, - end, - vm->usm.asid); - if (err) - goto wait; - ++fence_id; - } + xe_gt_tlb_inval_fence_init(tile->media_gt, + &fence[fence_id], true); + + err = xe_gt_tlb_inval_range(tile->media_gt, &fence[fence_id], + start, end, vm->usm.asid); + if (err) + goto wait; + ++fence_id; } wait: for (id = 0; id < fence_id; ++id) - xe_gt_tlb_invalidation_fence_wait(&fence[id]); + xe_gt_tlb_inval_fence_wait(&fence[id]); return err; } @@ -4147,8 +4143,8 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) xe_device_wmb(xe); - ret = xe_vm_range_tilemask_tlb_invalidation(xe_vma_vm(vma), xe_vma_start(vma), - xe_vma_end(vma), tile_mask); + ret = xe_vm_range_tilemask_tlb_inval(xe_vma_vm(vma), xe_vma_start(vma), + xe_vma_end(vma), tile_mask); /* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ WRITE_ONCE(vma->tile_invalidated, vma->tile_mask); diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 8afd2cd4c9dc..b3e5bec0fa58 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -236,8 +236,8 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, struct xe_svm_range *range); -int xe_vm_range_tilemask_tlb_invalidation(struct xe_vm *vm, u64 start, - u64 end, u8 tile_mask); +int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, + u64 end, u8 tile_mask); int xe_vm_invalidate_vma(struct xe_vma *vma); diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 7813bdedacaa..09c5783ee523 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -224,7 +224,7 @@ static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end) xe_device_wmb(vm->xe); - return xe_vm_range_tilemask_tlb_invalidation(vm, start, end, tile_mask); + return xe_vm_range_tilemask_tlb_inval(vm, start, end, tile_mask); } static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args) From 594bb930fc7dd40cea72c643047d642fadfc8b68 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:06 +0000 Subject: [PATCH 159/169] drm/xe: Add xe_tlb_inval structure Extract TLB invalidation state into a structure to decouple TLB invalidations from the GT, allowing the structure to be embedded anywhere in the driver. Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-5-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h | 34 ++++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_types.h | 33 ++------------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h index 919430359103..442f72b78ccf 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h @@ -6,10 +6,44 @@ #ifndef _XE_GT_TLB_INVAL_TYPES_H_ #define _XE_GT_TLB_INVAL_TYPES_H_ +#include #include struct xe_gt; +/** struct xe_tlb_inval - TLB invalidation client */ +struct xe_tlb_inval { + /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ +#define TLB_INVALIDATION_SEQNO_MAX 0x100000 + int seqno; + /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ + struct mutex seqno_lock; + /** + * @tlb_inval.seqno_recv: last received TLB invalidation seqno, + * protected by CT lock + */ + int seqno_recv; + /** + * @tlb_inval.pending_fences: list of pending fences waiting TLB + * invaliations, protected by CT lock + */ + struct list_head pending_fences; + /** + * @tlb_inval.pending_lock: protects @tlb_inval.pending_fences + * and updating @tlb_inval.seqno_recv. + */ + spinlock_t pending_lock; + /** + * @tlb_inval.fence_tdr: schedules a delayed call to + * xe_gt_tlb_fence_timeout after the timeut interval is over. + */ + struct delayed_work fence_tdr; + /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */ + struct workqueue_struct *job_wq; + /** @tlb_inval.lock: protects TLB invalidation fences */ + spinlock_t lock; +}; + /** * struct xe_gt_tlb_inval_fence - XE GT TLB invalidation fence * diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 85cfcc49472b..7dc5a3f310f1 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -12,6 +12,7 @@ #include "xe_gt_sriov_pf_types.h" #include "xe_gt_sriov_vf_types.h" #include "xe_gt_stats_types.h" +#include "xe_gt_tlb_inval_types.h" #include "xe_hw_engine_types.h" #include "xe_hw_fence_types.h" #include "xe_oa_types.h" @@ -186,37 +187,7 @@ struct xe_gt { } reset; /** @tlb_inval: TLB invalidation state */ - struct { - /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ -#define TLB_INVALIDATION_SEQNO_MAX 0x100000 - int seqno; - /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ - struct mutex seqno_lock; - /** - * @tlb_inval.seqno_recv: last received TLB invalidation seqno, - * protected by CT lock - */ - int seqno_recv; - /** - * @tlb_inval.pending_fences: list of pending fences waiting TLB - * invaliations, protected by CT lock - */ - struct list_head pending_fences; - /** - * @tlb_inval.pending_lock: protects @tlb_inval.pending_fences - * and updating @tlb_inval.seqno_recv. - */ - spinlock_t pending_lock; - /** - * @tlb_inval.fence_tdr: schedules a delayed call to - * xe_gt_tlb_fence_timeout after the timeut interval is over. - */ - struct delayed_work fence_tdr; - /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */ - struct workqueue_struct *job_wq; - /** @tlb_inval.lock: protects TLB invalidation fences */ - spinlock_t lock; - } tlb_inval; + struct xe_tlb_inval tlb_inval; /** * @ccs_mode: Number of compute engines enabled. From 6d1e452e0948fd69ec3aeea0e52807152dc93c2d Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:07 +0000 Subject: [PATCH 160/169] drm/xe: Add xe_gt_tlb_invalidation_done_handler Decouple GT TLB seqno handling from G2H handler. v2: - Add kernel doc Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-6-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_gt_tlb_inval.c | 47 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval.c b/drivers/gpu/drm/xe/xe_gt_tlb_inval.c index 1571fd917830..37b3b45ec230 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_gt_tlb_inval.c @@ -506,27 +506,18 @@ void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm) } /** - * xe_guc_tlb_inval_done_handler - TLB invalidation done handler - * @guc: guc - * @msg: message indicating TLB invalidation done - * @len: length of message + * xe_gt_tlb_inval_done_handler - GT TLB invalidation done handler + * @gt: gt + * @seqno: seqno of invalidation that is done * - * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any - * invalidation fences for seqno. Algorithm for this depends on seqno being - * received in-order and asserts this assumption. - * - * Return: 0 on success, -EPROTO for malformed messages. + * Update recv seqno, signal any GT TLB invalidation fences, and restart TDR */ -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +static void xe_gt_tlb_inval_done_handler(struct xe_gt *gt, int seqno) { - struct xe_gt *gt = guc_to_gt(guc); struct xe_device *xe = gt_to_xe(gt); struct xe_gt_tlb_inval_fence *fence, *next; unsigned long flags; - if (unlikely(len != 1)) - return -EPROTO; - /* * This can also be run both directly from the IRQ handler and also in * process_g2h_msg(). Only one may process any individual CT message, @@ -543,12 +534,12 @@ int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) * process_g2h_msg(). */ spin_lock_irqsave(>->tlb_inval.pending_lock, flags); - if (tlb_inval_seqno_past(gt, msg[0])) { + if (tlb_inval_seqno_past(gt, seqno)) { spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); - return 0; + return; } - WRITE_ONCE(gt->tlb_inval.seqno_recv, msg[0]); + WRITE_ONCE(gt->tlb_inval.seqno_recv, seqno); list_for_each_entry_safe(fence, next, >->tlb_inval.pending_fences, link) { @@ -568,6 +559,28 @@ int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) cancel_delayed_work(>->tlb_inval.fence_tdr); spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); +} + +/** + * xe_guc_tlb_inval_done_handler - TLB invalidation done handler + * @guc: guc + * @msg: message indicating TLB invalidation done + * @len: length of message + * + * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any + * invalidation fences for seqno. Algorithm for this depends on seqno being + * received in-order and asserts this assumption. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + + if (unlikely(len != 1)) + return -EPROTO; + + xe_gt_tlb_inval_done_handler(gt, msg[0]); return 0; } From 15366239e2130ebe662351c0995590f2af2ba3b3 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:08 +0000 Subject: [PATCH 161/169] drm/xe: Decouple TLB invalidations from GT Decouple TLB invalidations from the GT by updating the TLB invalidation layer to accept a `struct xe_tlb_inval` instead of a `struct xe_gt`. Also, rename *gt_tlb* to *tlb*. The internals of the TLB invalidation code still operate on a GT, but this is now hidden from the rest of the driver. Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-7-stuart.summers@intel.com --- drivers/gpu/drm/xe/Makefile | 4 +- drivers/gpu/drm/xe/xe_ggtt.c | 4 +- drivers/gpu/drm/xe/xe_gt.c | 6 +- drivers/gpu/drm/xe/xe_gt_tlb_inval.h | 41 ----- drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h | 34 ---- drivers/gpu/drm/xe/xe_gt_types.h | 2 +- drivers/gpu/drm/xe/xe_guc_ct.c | 2 +- drivers/gpu/drm/xe/xe_lmtt.c | 12 +- drivers/gpu/drm/xe/xe_migrate.h | 10 +- drivers/gpu/drm/xe/xe_pt.c | 63 ++++--- drivers/gpu/drm/xe/xe_svm.c | 1 - .../xe/{xe_gt_tlb_inval.c => xe_tlb_inval.c} | 142 +++++++++------- drivers/gpu/drm/xe/xe_tlb_inval.h | 41 +++++ ..._gt_tlb_inval_job.c => xe_tlb_inval_job.c} | 154 +++++++++--------- drivers/gpu/drm/xe/xe_tlb_inval_job.h | 33 ++++ ...tlb_inval_types.h => xe_tlb_inval_types.h} | 35 ++-- drivers/gpu/drm/xe/xe_trace.h | 24 +-- drivers/gpu/drm/xe/xe_vm.c | 26 +-- 18 files changed, 330 insertions(+), 304 deletions(-) delete mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_inval.h delete mode 100644 drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h rename drivers/gpu/drm/xe/{xe_gt_tlb_inval.c => xe_tlb_inval.c} (80%) create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval.h rename drivers/gpu/drm/xe/{xe_gt_tlb_inval_job.c => xe_tlb_inval_job.c} (51%) create mode 100644 drivers/gpu/drm/xe/xe_tlb_inval_job.h rename drivers/gpu/drm/xe/{xe_gt_tlb_inval_types.h => xe_tlb_inval_types.h} (56%) diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index a1e4b3b1a1b6..2b67debb8bc4 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -61,8 +61,6 @@ xe-y += xe_bb.o \ xe_gt_pagefault.o \ xe_gt_sysfs.o \ xe_gt_throttle.o \ - xe_gt_tlb_inval.o \ - xe_gt_tlb_inval_job.o \ xe_gt_topology.o \ xe_guc.o \ xe_guc_ads.o \ @@ -119,6 +117,8 @@ xe-y += xe_bb.o \ xe_sync.o \ xe_tile.o \ xe_tile_sysfs.o \ + xe_tlb_inval.o \ + xe_tlb_inval_job.o \ xe_trace.o \ xe_trace_bo.o \ xe_trace_guc.o \ diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index c3e46c270117..71c7690a92b3 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -23,13 +23,13 @@ #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_printk.h" -#include "xe_gt_tlb_inval.h" #include "xe_map.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_res_cursor.h" #include "xe_sriov.h" #include "xe_tile_sriov_vf.h" +#include "xe_tlb_inval.h" #include "xe_wa.h" #include "xe_wopcm.h" @@ -438,7 +438,7 @@ static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) if (!gt) return; - err = xe_gt_tlb_inval_ggtt(gt); + err = xe_tlb_inval_ggtt(>->tlb_inval); xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err)); } diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 9a4639732bd7..67ee7cdbd6ec 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -37,7 +37,6 @@ #include "xe_gt_sriov_pf.h" #include "xe_gt_sriov_vf.h" #include "xe_gt_sysfs.h" -#include "xe_gt_tlb_inval.h" #include "xe_gt_topology.h" #include "xe_guc_exec_queue_types.h" #include "xe_guc_pc.h" @@ -58,6 +57,7 @@ #include "xe_sa.h" #include "xe_sched_job.h" #include "xe_sriov.h" +#include "xe_tlb_inval.h" #include "xe_tuning.h" #include "xe_uc.h" #include "xe_uc_fw.h" @@ -852,7 +852,7 @@ static int gt_reset(struct xe_gt *gt) xe_uc_stop(>->uc); - xe_gt_tlb_inval_reset(gt); + xe_tlb_inval_reset(>->tlb_inval); err = do_gt_reset(gt); if (err) @@ -1066,5 +1066,5 @@ void xe_gt_declare_wedged(struct xe_gt *gt) xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode); xe_uc_declare_wedged(>->uc); - xe_gt_tlb_inval_reset(gt); + xe_tlb_inval_reset(>->tlb_inval); } diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval.h deleted file mode 100644 index b1258ac4e8fb..000000000000 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef _XE_GT_TLB_INVAL_H_ -#define _XE_GT_TLB_INVAL_H_ - -#include - -#include "xe_gt_tlb_inval_types.h" - -struct xe_gt; -struct xe_guc; -struct xe_vm; -struct xe_vma; - -int xe_gt_tlb_inval_init_early(struct xe_gt *gt); -void xe_gt_tlb_inval_fini(struct xe_gt *gt); - -void xe_gt_tlb_inval_reset(struct xe_gt *gt); -int xe_gt_tlb_inval_ggtt(struct xe_gt *gt); -void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm); -int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence); -int xe_gt_tlb_inval_range(struct xe_gt *gt, - struct xe_gt_tlb_inval_fence *fence, - u64 start, u64 end, u32 asid); -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); - -void xe_gt_tlb_inval_fence_init(struct xe_gt *gt, - struct xe_gt_tlb_inval_fence *fence, - bool stack); -void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence); - -static inline void -xe_gt_tlb_inval_fence_wait(struct xe_gt_tlb_inval_fence *fence) -{ - dma_fence_wait(&fence->base, false); -} - -#endif /* _XE_GT_TLB_INVAL_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h deleted file mode 100644 index 883896194a34..000000000000 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2025 Intel Corporation - */ - -#ifndef _XE_GT_TLB_INVAL_JOB_H_ -#define _XE_GT_TLB_INVAL_JOB_H_ - -#include - -struct dma_fence; -struct drm_sched_job; -struct kref; -struct xe_exec_queue; -struct xe_gt; -struct xe_gt_tlb_inval_job; -struct xe_migrate; - -struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, - struct xe_gt *gt, - u64 start, u64 end, - u32 asid); - -int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job); - -struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, - struct xe_migrate *m, - struct dma_fence *fence); - -void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job); - -void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job); - -#endif diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 7dc5a3f310f1..66158105aca5 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -12,12 +12,12 @@ #include "xe_gt_sriov_pf_types.h" #include "xe_gt_sriov_vf_types.h" #include "xe_gt_stats_types.h" -#include "xe_gt_tlb_inval_types.h" #include "xe_hw_engine_types.h" #include "xe_hw_fence_types.h" #include "xe_oa_types.h" #include "xe_reg_sr_types.h" #include "xe_sa_types.h" +#include "xe_tlb_inval_types.h" #include "xe_uc_types.h" struct xe_exec_queue_ops; diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 9131d121d941..5f38041cff4c 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -26,13 +26,13 @@ #include "xe_gt_sriov_pf_control.h" #include "xe_gt_sriov_pf_monitor.h" #include "xe_gt_sriov_printk.h" -#include "xe_gt_tlb_inval.h" #include "xe_guc.h" #include "xe_guc_log.h" #include "xe_guc_relay.h" #include "xe_guc_submit.h" #include "xe_map.h" #include "xe_pm.h" +#include "xe_tlb_inval.h" #include "xe_trace_guc.h" static void receive_g2h(struct xe_guc_ct *ct); diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c index e5aba03ff8ac..f2bfbfa3efa1 100644 --- a/drivers/gpu/drm/xe/xe_lmtt.c +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -11,7 +11,7 @@ #include "xe_assert.h" #include "xe_bo.h" -#include "xe_gt_tlb_inval.h" +#include "xe_tlb_inval.h" #include "xe_lmtt.h" #include "xe_map.h" #include "xe_mmio.h" @@ -228,8 +228,8 @@ void xe_lmtt_init_hw(struct xe_lmtt *lmtt) static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) { - struct xe_gt_tlb_inval_fence fences[XE_MAX_GT_PER_TILE]; - struct xe_gt_tlb_inval_fence *fence = fences; + struct xe_tlb_inval_fence fences[XE_MAX_GT_PER_TILE]; + struct xe_tlb_inval_fence *fence = fences; struct xe_tile *tile = lmtt_to_tile(lmtt); struct xe_gt *gt; int result = 0; @@ -237,8 +237,8 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) u8 id; for_each_gt_on_tile(gt, tile, id) { - xe_gt_tlb_inval_fence_init(gt, fence, true); - err = xe_gt_tlb_inval_all(gt, fence); + xe_tlb_inval_fence_init(>->tlb_inval, fence, true); + err = xe_tlb_inval_all(>->tlb_inval, fence); result = result ?: err; fence++; } @@ -252,7 +252,7 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) */ fence = fences; for_each_gt_on_tile(gt, tile, id) - xe_gt_tlb_inval_fence_wait(fence++); + xe_tlb_inval_fence_wait(fence++); return result; } diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 8978d2cc1a75..4fad324b6253 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -15,7 +15,7 @@ struct ttm_resource; struct xe_bo; struct xe_gt; -struct xe_gt_tlb_inval_job; +struct xe_tlb_inval_job; struct xe_exec_queue; struct xe_migrate; struct xe_migrate_pt_update; @@ -94,13 +94,13 @@ struct xe_migrate_pt_update { /** @job: The job if a GPU page-table update. NULL otherwise */ struct xe_sched_job *job; /** - * @ijob: The GT TLB invalidation job for primary tile. NULL otherwise + * @ijob: The TLB invalidation job for primary GT. NULL otherwise */ - struct xe_gt_tlb_inval_job *ijob; + struct xe_tlb_inval_job *ijob; /** - * @mjob: The GT TLB invalidation job for media tile. NULL otherwise + * @mjob: The TLB invalidation job for media GT. NULL otherwise */ - struct xe_gt_tlb_inval_job *mjob; + struct xe_tlb_inval_job *mjob; /** @tile_id: Tile ID of the update */ u8 tile_id; }; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index c0a70c80dff9..c129048a9a09 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -13,7 +13,7 @@ #include "xe_drm_client.h" #include "xe_exec_queue.h" #include "xe_gt.h" -#include "xe_gt_tlb_inval_job.h" +#include "xe_tlb_inval_job.h" #include "xe_migrate.h" #include "xe_pt_types.h" #include "xe_pt_walk.h" @@ -21,6 +21,7 @@ #include "xe_sched_job.h" #include "xe_sync.h" #include "xe_svm.h" +#include "xe_tlb_inval_job.h" #include "xe_trace.h" #include "xe_ttm_stolen_mgr.h" #include "xe_vm.h" @@ -1276,8 +1277,8 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, } static int xe_pt_vm_dependencies(struct xe_sched_job *job, - struct xe_gt_tlb_inval_job *ijob, - struct xe_gt_tlb_inval_job *mjob, + struct xe_tlb_inval_job *ijob, + struct xe_tlb_inval_job *mjob, struct xe_vm *vm, struct xe_vma_ops *vops, struct xe_vm_pgtable_update_ops *pt_update_ops, @@ -1347,13 +1348,13 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, if (job) { if (ijob) { - err = xe_gt_tlb_inval_job_alloc_dep(ijob); + err = xe_tlb_inval_job_alloc_dep(ijob); if (err) return err; } if (mjob) { - err = xe_gt_tlb_inval_job_alloc_dep(mjob); + err = xe_tlb_inval_job_alloc_dep(mjob); if (err) return err; } @@ -2353,6 +2354,15 @@ static const struct xe_migrate_pt_update_ops svm_migrate_ops = { static const struct xe_migrate_pt_update_ops svm_migrate_ops; #endif +static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q, + struct xe_gt *gt) +{ + if (xe_gt_is_media_type(gt)) + return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler; + + return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler; +} + /** * xe_pt_update_ops_run() - Run PT update operations * @tile: Tile of PT update operations @@ -2371,7 +2381,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id]; struct dma_fence *fence, *ifence, *mfence; - struct xe_gt_tlb_inval_job *ijob = NULL, *mjob = NULL; + struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; struct xe_range_fence *rfence; @@ -2403,11 +2413,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) #endif if (pt_update_ops->needs_invalidation) { - ijob = xe_gt_tlb_inval_job_create(pt_update_ops->q, - tile->primary_gt, - pt_update_ops->start, - pt_update_ops->last, - vm->usm.asid); + struct xe_exec_queue *q = pt_update_ops->q; + struct xe_dep_scheduler *dep_scheduler = + to_dep_scheduler(q, tile->primary_gt); + + ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval, + dep_scheduler, + pt_update_ops->start, + pt_update_ops->last, + vm->usm.asid); if (IS_ERR(ijob)) { err = PTR_ERR(ijob); goto kill_vm_tile1; @@ -2415,11 +2429,14 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) update.ijob = ijob; if (tile->media_gt) { - mjob = xe_gt_tlb_inval_job_create(pt_update_ops->q, - tile->media_gt, - pt_update_ops->start, - pt_update_ops->last, - vm->usm.asid); + dep_scheduler = to_dep_scheduler(q, tile->media_gt); + + mjob = xe_tlb_inval_job_create(q, + &tile->media_gt->tlb_inval, + dep_scheduler, + pt_update_ops->start, + pt_update_ops->last, + vm->usm.asid); if (IS_ERR(mjob)) { err = PTR_ERR(mjob); goto free_ijob; @@ -2470,13 +2487,13 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) if (ijob) { struct dma_fence *__fence; - ifence = xe_gt_tlb_inval_job_push(ijob, tile->migrate, fence); + ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence); __fence = ifence; if (mjob) { fences[0] = ifence; - mfence = xe_gt_tlb_inval_job_push(mjob, tile->migrate, - fence); + mfence = xe_tlb_inval_job_push(mjob, tile->migrate, + fence); fences[1] = mfence; dma_fence_array_init(cf, 2, fences, @@ -2519,8 +2536,8 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) if (pt_update_ops->needs_userptr_lock) up_read(&vm->userptr.notifier_lock); - xe_gt_tlb_inval_job_put(mjob); - xe_gt_tlb_inval_job_put(ijob); + xe_tlb_inval_job_put(mjob); + xe_tlb_inval_job_put(ijob); return fence; @@ -2529,8 +2546,8 @@ free_rfence: free_ijob: kfree(cf); kfree(fences); - xe_gt_tlb_inval_job_put(mjob); - xe_gt_tlb_inval_job_put(ijob); + xe_tlb_inval_job_put(mjob); + xe_tlb_inval_job_put(ijob); kill_vm_tile1: if (err != -EAGAIN && err != -ENODATA && tile->id) xe_vm_kill(vops->vm, false); diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index babc64485b52..76c6d74c1208 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -7,7 +7,6 @@ #include "xe_bo.h" #include "xe_gt_stats.h" -#include "xe_gt_tlb_inval.h" #include "xe_migrate.h" #include "xe_module.h" #include "xe_pm.h" diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c similarity index 80% rename from drivers/gpu/drm/xe/xe_gt_tlb_inval.c rename to drivers/gpu/drm/xe/xe_tlb_inval.c index 37b3b45ec230..f4b7c0c74894 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -13,7 +13,7 @@ #include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_gt_stats.h" -#include "xe_gt_tlb_inval.h" +#include "xe_tlb_inval.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_sriov.h" @@ -38,40 +38,47 @@ static long tlb_timeout_jiffies(struct xe_gt *gt) return hw_tlb_timeout + 2 * delay; } -static void xe_gt_tlb_inval_fence_fini(struct xe_gt_tlb_inval_fence *fence) +static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence) { - if (WARN_ON_ONCE(!fence->gt)) + struct xe_gt *gt; + + if (WARN_ON_ONCE(!fence->tlb_inval)) return; - xe_pm_runtime_put(gt_to_xe(fence->gt)); - fence->gt = NULL; /* fini() should be called once */ + gt = fence->tlb_inval->private; + + xe_pm_runtime_put(gt_to_xe(gt)); + fence->tlb_inval = NULL; /* fini() should be called once */ } static void -__inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence) +__inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) { bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags); - trace_xe_gt_tlb_inval_fence_signal(xe, fence); - xe_gt_tlb_inval_fence_fini(fence); + trace_xe_tlb_inval_fence_signal(xe, fence); + xe_tlb_inval_fence_fini(fence); dma_fence_signal(&fence->base); if (!stack) dma_fence_put(&fence->base); } static void -inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence) +inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) { list_del(&fence->link); __inval_fence_signal(xe, fence); } -void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence) +void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence) { - if (WARN_ON_ONCE(!fence->gt)) + struct xe_gt *gt; + + if (WARN_ON_ONCE(!fence->tlb_inval)) return; - __inval_fence_signal(gt_to_xe(fence->gt), fence); + gt = fence->tlb_inval->private; + __inval_fence_signal(gt_to_xe(gt), fence); } static void xe_gt_tlb_fence_timeout(struct work_struct *work) @@ -79,7 +86,7 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) struct xe_gt *gt = container_of(work, struct xe_gt, tlb_inval.fence_tdr.work); struct xe_device *xe = gt_to_xe(gt); - struct xe_gt_tlb_inval_fence *fence, *next; + struct xe_tlb_inval_fence *fence, *next; LNL_FLUSH_WORK(>->uc.guc.ct.g2h_worker); @@ -92,7 +99,7 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt)) break; - trace_xe_gt_tlb_inval_fence_timeout(xe, fence); + trace_xe_tlb_inval_fence_timeout(xe, fence); xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d", fence->seqno, gt->tlb_inval.seqno_recv); @@ -110,7 +117,7 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) * xe_gt_tlb_inval_init_early - Initialize GT TLB invalidation state * @gt: GT structure * - * Initialize GT TLB invalidation state, purely software initialization, should + * Initialize TLB invalidation state, purely software initialization, should * be called once during driver load. * * Return: 0 on success, negative error code on error. @@ -120,6 +127,7 @@ int xe_gt_tlb_inval_init_early(struct xe_gt *gt) struct xe_device *xe = gt_to_xe(gt); int err; + gt->tlb_inval.private = gt; gt->tlb_inval.seqno = 1; INIT_LIST_HEAD(>->tlb_inval.pending_fences); spin_lock_init(>->tlb_inval.pending_lock); @@ -141,14 +149,15 @@ int xe_gt_tlb_inval_init_early(struct xe_gt *gt) } /** - * xe_gt_tlb_inval_reset - Initialize GT TLB invalidation reset - * @gt: GT structure + * xe_tlb_inval_reset - Initialize TLB invalidation reset + * @tlb_inval: TLB invalidation client * * Signal any pending invalidation fences, should be called during a GT reset */ -void xe_gt_tlb_inval_reset(struct xe_gt *gt) +void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) { - struct xe_gt_tlb_inval_fence *fence, *next; + struct xe_gt *gt = tlb_inval->private; + struct xe_tlb_inval_fence *fence, *next; int pending_seqno; /* @@ -213,7 +222,7 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) } static int send_tlb_inval(struct xe_guc *guc, - struct xe_gt_tlb_inval_fence *fence, + struct xe_tlb_inval_fence *fence, u32 *action, int len) { struct xe_gt *gt = guc_to_gt(guc); @@ -232,7 +241,7 @@ static int send_tlb_inval(struct xe_guc *guc, mutex_lock(>->tlb_inval.seqno_lock); seqno = gt->tlb_inval.seqno; fence->seqno = seqno; - trace_xe_gt_tlb_inval_fence_send(xe, fence); + trace_xe_tlb_inval_fence_send(xe, fence); action[1] = seqno; ret = xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_TLB_INVALIDATE, 1); @@ -277,7 +286,7 @@ static int send_tlb_inval(struct xe_guc *guc, XE_GUC_TLB_INVAL_FLUSH_CACHE) /** - * xe_gt_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC + * xe_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC * @gt: GT structure * @fence: invalidation fence which will be signal on TLB invalidation * completion @@ -287,8 +296,8 @@ static int send_tlb_inval(struct xe_guc *guc, * * Return: 0 on success, negative error code on error */ -static int xe_gt_tlb_inval_guc(struct xe_gt *gt, - struct xe_gt_tlb_inval_fence *fence) +static int xe_tlb_inval_guc(struct xe_gt *gt, + struct xe_tlb_inval_fence *fence) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION, @@ -309,30 +318,31 @@ static int xe_gt_tlb_inval_guc(struct xe_gt *gt, } /** - * xe_gt_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT - * @gt: GT structure + * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT + * @tlb_inval: TLB invalidation client * * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is * synchronous. * * Return: 0 on success, negative error code on error */ -int xe_gt_tlb_inval_ggtt(struct xe_gt *gt) +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) { + struct xe_gt *gt = tlb_inval->private; struct xe_device *xe = gt_to_xe(gt); unsigned int fw_ref; if (xe_guc_ct_enabled(>->uc.guc.ct) && gt->uc.guc.submission_state.enabled) { - struct xe_gt_tlb_inval_fence fence; + struct xe_tlb_inval_fence fence; int ret; - xe_gt_tlb_inval_fence_init(gt, &fence, true); - ret = xe_gt_tlb_inval_guc(gt, &fence); + xe_tlb_inval_fence_init(tlb_inval, &fence, true); + ret = xe_tlb_inval_guc(gt, &fence); if (ret) return ret; - xe_gt_tlb_inval_fence_wait(&fence); + xe_tlb_inval_fence_wait(&fence); } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { struct xe_mmio *mmio = >->mmio; @@ -355,14 +365,17 @@ int xe_gt_tlb_inval_ggtt(struct xe_gt *gt) return 0; } -static int send_tlb_inval_all(struct xe_gt *gt, - struct xe_gt_tlb_inval_fence *fence) +static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION_ALL, 0, /* seqno, replaced in send_tlb_inval */ MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), }; + struct xe_gt *gt = tlb_inval->private; + + xe_gt_assert(gt, fence); return send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); } @@ -370,19 +383,19 @@ static int send_tlb_inval_all(struct xe_gt *gt, /** * xe_gt_tlb_invalidation_all - Invalidate all TLBs across PF and all VFs. * @gt: the &xe_gt structure - * @fence: the &xe_gt_tlb_inval_fence to be signaled on completion + * @fence: the &xe_tlb_inval_fence to be signaled on completion * * Send a request to invalidate all TLBs across PF and all VFs. * * Return: 0 on success, negative error code on error */ -int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence) +int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence) { + struct xe_gt *gt = tlb_inval->private; int err; - xe_gt_assert(gt, gt == fence->gt); - - err = send_tlb_inval_all(gt, fence); + err = send_tlb_inval_all(tlb_inval, fence); if (err) xe_gt_err(gt, "TLB invalidation request failed (%pe)", ERR_PTR(err)); @@ -397,9 +410,8 @@ int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence) #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) /** - * xe_gt_tlb_inval_range - Issue a TLB invalidation on this GT for an address range - * - * @gt: GT structure + * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range + * @tlb_inval: TLB invalidation client * @fence: invalidation fence which will be signal on TLB invalidation * completion * @start: start address @@ -412,9 +424,11 @@ int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence) * * Return: Negative error code on error, 0 on success */ -int xe_gt_tlb_inval_range(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence, - u64 start, u64 end, u32 asid) +int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, u64 start, u64 end, + u32 asid) { + struct xe_gt *gt = tlb_inval->private; struct xe_device *xe = gt_to_xe(gt); #define MAX_TLB_INVALIDATION_LEN 7 u32 action[MAX_TLB_INVALIDATION_LEN]; @@ -484,38 +498,38 @@ int xe_gt_tlb_inval_range(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence, } /** - * xe_gt_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM - * @gt: graphics tile + * xe_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM + * @tlb_inval: TLB invalidation client * @vm: VM to invalidate * * Invalidate entire VM's address space */ -void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm) +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm) { - struct xe_gt_tlb_inval_fence fence; + struct xe_tlb_inval_fence fence; u64 range = 1ull << vm->xe->info.va_bits; int ret; - xe_gt_tlb_inval_fence_init(gt, &fence, true); + xe_tlb_inval_fence_init(tlb_inval, &fence, true); - ret = xe_gt_tlb_inval_range(gt, &fence, 0, range, vm->usm.asid); + ret = xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid); if (ret < 0) return; - xe_gt_tlb_inval_fence_wait(&fence); + xe_tlb_inval_fence_wait(&fence); } /** - * xe_gt_tlb_inval_done_handler - GT TLB invalidation done handler + * xe_tlb_inval_done_handler - TLB invalidation done handler * @gt: gt * @seqno: seqno of invalidation that is done * - * Update recv seqno, signal any GT TLB invalidation fences, and restart TDR + * Update recv seqno, signal any TLB invalidation fences, and restart TDR */ -static void xe_gt_tlb_inval_done_handler(struct xe_gt *gt, int seqno) +static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno) { struct xe_device *xe = gt_to_xe(gt); - struct xe_gt_tlb_inval_fence *fence, *next; + struct xe_tlb_inval_fence *fence, *next; unsigned long flags; /* @@ -543,7 +557,7 @@ static void xe_gt_tlb_inval_done_handler(struct xe_gt *gt, int seqno) list_for_each_entry_safe(fence, next, >->tlb_inval.pending_fences, link) { - trace_xe_gt_tlb_inval_fence_recv(xe, fence); + trace_xe_tlb_inval_fence_recv(xe, fence); if (!tlb_inval_seqno_past(gt, fence->seqno)) break; @@ -580,7 +594,7 @@ int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) if (unlikely(len != 1)) return -EPROTO; - xe_gt_tlb_inval_done_handler(gt, msg[0]); + xe_tlb_inval_done_handler(gt, msg[0]); return 0; } @@ -603,19 +617,21 @@ static const struct dma_fence_ops inval_fence_ops = { }; /** - * xe_gt_tlb_inval_fence_init - Initialize TLB invalidation fence - * @gt: GT + * xe_tlb_inval_fence_init - Initialize TLB invalidation fence + * @tlb_inval: TLB invalidation client * @fence: TLB invalidation fence to initialize * @stack: fence is stack variable * - * Initialize TLB invalidation fence for use. xe_gt_tlb_inval_fence_fini + * Initialize TLB invalidation fence for use. xe_tlb_inval_fence_fini * will be automatically called when fence is signalled (all fences must signal), * even on error. */ -void xe_gt_tlb_inval_fence_init(struct xe_gt *gt, - struct xe_gt_tlb_inval_fence *fence, - bool stack) +void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + bool stack) { + struct xe_gt *gt = tlb_inval->private; + xe_pm_runtime_get_noresume(gt_to_xe(gt)); spin_lock_irq(>->tlb_inval.lock); @@ -628,5 +644,5 @@ void xe_gt_tlb_inval_fence_init(struct xe_gt *gt, set_bit(FENCE_STACK_BIT, &fence->base.flags); else dma_fence_get(&fence->base); - fence->gt = gt; + fence->tlb_inval = tlb_inval; } diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h new file mode 100644 index 000000000000..ab6f769c50be --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TLB_INVAL_H_ +#define _XE_TLB_INVAL_H_ + +#include + +#include "xe_tlb_inval_types.h" + +struct xe_gt; +struct xe_guc; +struct xe_vm; + +int xe_gt_tlb_inval_init_early(struct xe_gt *gt); +void xe_gt_tlb_inval_fini(struct xe_gt *gt); + +void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval); +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm); +int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence); +int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + u64 start, u64 end, u32 asid); +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); + +void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, + bool stack); +void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence); + +static inline void +xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence) +{ + dma_fence_wait(&fence->base, false); +} + +#endif /* _XE_TLB_INVAL_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c similarity index 51% rename from drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c rename to drivers/gpu/drm/xe/xe_tlb_inval_job.c index 41e0ea92ea5a..492def04a559 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -3,21 +3,22 @@ * Copyright © 2025 Intel Corporation */ +#include "xe_assert.h" #include "xe_dep_job_types.h" #include "xe_dep_scheduler.h" #include "xe_exec_queue.h" -#include "xe_gt.h" -#include "xe_gt_tlb_inval.h" -#include "xe_gt_tlb_inval_job.h" +#include "xe_gt_types.h" +#include "xe_tlb_inval.h" +#include "xe_tlb_inval_job.h" #include "xe_migrate.h" #include "xe_pm.h" -/** struct xe_gt_tlb_inval_job - GT TLB invalidation job */ -struct xe_gt_tlb_inval_job { +/** struct xe_tlb_inval_job - TLB invalidation job */ +struct xe_tlb_inval_job { /** @dep: base generic dependency Xe job */ struct xe_dep_job dep; - /** @gt: GT to invalidate */ - struct xe_gt *gt; + /** @tlb_inval: TLB invalidation client */ + struct xe_tlb_inval *tlb_inval; /** @q: exec queue issuing the invalidate */ struct xe_exec_queue *q; /** @refcount: ref count of this job */ @@ -37,63 +38,56 @@ struct xe_gt_tlb_inval_job { bool fence_armed; }; -static struct dma_fence *xe_gt_tlb_inval_job_run(struct xe_dep_job *dep_job) +static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job) { - struct xe_gt_tlb_inval_job *job = + struct xe_tlb_inval_job *job = container_of(dep_job, typeof(*job), dep); - struct xe_gt_tlb_inval_fence *ifence = + struct xe_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); - xe_gt_tlb_inval_range(job->gt, ifence, job->start, - job->end, job->asid); + xe_tlb_inval_range(job->tlb_inval, ifence, job->start, + job->end, job->asid); return job->fence; } -static void xe_gt_tlb_inval_job_free(struct xe_dep_job *dep_job) +static void xe_tlb_inval_job_free(struct xe_dep_job *dep_job) { - struct xe_gt_tlb_inval_job *job = + struct xe_tlb_inval_job *job = container_of(dep_job, typeof(*job), dep); - /* Pairs with get in xe_gt_tlb_inval_job_push */ - xe_gt_tlb_inval_job_put(job); + /* Pairs with get in xe_tlb_inval_job_push */ + xe_tlb_inval_job_put(job); } static const struct xe_dep_job_ops dep_job_ops = { - .run_job = xe_gt_tlb_inval_job_run, - .free_job = xe_gt_tlb_inval_job_free, + .run_job = xe_tlb_inval_job_run, + .free_job = xe_tlb_inval_job_free, }; -static int xe_gt_tlb_inval_context(struct xe_gt *gt) -{ - return xe_gt_is_media_type(gt) ? XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT : - XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; -} - /** - * xe_gt_tlb_inval_job_create() - GT TLB invalidation job create - * @gt: GT to invalidate + * xe_tlb_inval_job_create() - TLB invalidation job create * @q: exec queue issuing the invalidate + * @tlb_inval: TLB invalidation client + * @dep_scheduler: Dependency scheduler for job * @start: Start address to invalidate * @end: End address to invalidate * @asid: Address space ID to invalidate * - * Create a GT TLB invalidation job and initialize internal fields. The caller is + * Create a TLB invalidation job and initialize internal fields. The caller is * responsible for releasing the creation reference. * - * Return: GT TLB invalidation job object on success, ERR_PTR failure + * Return: TLB invalidation job object on success, ERR_PTR failure */ -struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, - struct xe_gt *gt, - u64 start, u64 end, - u32 asid) +struct xe_tlb_inval_job * +xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, + struct xe_dep_scheduler *dep_scheduler, u64 start, + u64 end, u32 asid) { - struct xe_gt_tlb_inval_job *job; - struct xe_dep_scheduler *dep_scheduler = - q->tlb_inval[xe_gt_tlb_inval_context(gt)].dep_scheduler; + struct xe_tlb_inval_job *job; struct drm_sched_entity *entity = xe_dep_scheduler_entity(dep_scheduler); - struct xe_gt_tlb_inval_fence *ifence; + struct xe_tlb_inval_fence *ifence; int err; job = kmalloc(sizeof(*job), GFP_KERNEL); @@ -101,14 +95,14 @@ struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, return ERR_PTR(-ENOMEM); job->q = q; - job->gt = gt; + job->tlb_inval = tlb_inval; job->start = start; job->end = end; job->asid = asid; job->fence_armed = false; job->dep.ops = &dep_job_ops; kref_init(&job->refcount); - xe_exec_queue_get(q); /* Pairs with put in xe_gt_tlb_inval_job_destroy */ + xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */ ifence = kmalloc(sizeof(*ifence), GFP_KERNEL); if (!ifence) { @@ -122,8 +116,8 @@ struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q, if (err) goto err_fence; - /* Pairs with put in xe_gt_tlb_inval_job_destroy */ - xe_pm_runtime_get_noresume(gt_to_xe(job->gt)); + /* Pairs with put in xe_tlb_inval_job_destroy */ + xe_pm_runtime_get_noresume(gt_to_xe(q->gt)); return job; @@ -136,40 +130,40 @@ err_job: return ERR_PTR(err); } -static void xe_gt_tlb_inval_job_destroy(struct kref *ref) +static void xe_tlb_inval_job_destroy(struct kref *ref) { - struct xe_gt_tlb_inval_job *job = container_of(ref, typeof(*job), - refcount); - struct xe_gt_tlb_inval_fence *ifence = + struct xe_tlb_inval_job *job = container_of(ref, typeof(*job), + refcount); + struct xe_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); - struct xe_device *xe = gt_to_xe(job->gt); struct xe_exec_queue *q = job->q; + struct xe_device *xe = gt_to_xe(q->gt); if (!job->fence_armed) kfree(ifence); else - /* Ref from xe_gt_tlb_inval_fence_init */ + /* Ref from xe_tlb_inval_fence_init */ dma_fence_put(job->fence); drm_sched_job_cleanup(&job->dep.drm); kfree(job); - xe_exec_queue_put(q); /* Pairs with get from xe_gt_tlb_inval_job_create */ - xe_pm_runtime_put(xe); /* Pairs with get from xe_gt_tlb_inval_job_create */ + xe_exec_queue_put(q); /* Pairs with get from xe_tlb_inval_job_create */ + xe_pm_runtime_put(xe); /* Pairs with get from xe_tlb_inval_job_create */ } /** - * xe_gt_tlb_inval_alloc_dep() - GT TLB invalidation job alloc dependency - * @job: GT TLB invalidation job to alloc dependency for + * xe_tlb_inval_alloc_dep() - TLB invalidation job alloc dependency + * @job: TLB invalidation job to alloc dependency for * - * Allocate storage for a dependency in the GT TLB invalidation fence. This + * Allocate storage for a dependency in the TLB invalidation fence. This * function should be called at most once per job and must be paired with - * xe_gt_tlb_inval_job_push being called with a real fence. + * xe_tlb_inval_job_push being called with a real fence. * * Return: 0 on success, -errno on failure */ -int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job) +int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job) { - xe_assert(gt_to_xe(job->gt), !xa_load(&job->dep.drm.dependencies, 0)); + xe_assert(gt_to_xe(job->q->gt), !xa_load(&job->dep.drm.dependencies, 0)); might_alloc(GFP_KERNEL); return drm_sched_job_add_dependency(&job->dep.drm, @@ -177,24 +171,24 @@ int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job) } /** - * xe_gt_tlb_inval_job_push() - GT TLB invalidation job push - * @job: GT TLB invalidation job to push + * xe_tlb_inval_job_push() - TLB invalidation job push + * @job: TLB invalidation job to push * @m: The migration object being used - * @fence: Dependency for GT TLB invalidation job + * @fence: Dependency for TLB invalidation job * - * Pushes a GT TLB invalidation job for execution, using @fence as a dependency. - * Storage for @fence must be preallocated with xe_gt_tlb_inval_job_alloc_dep + * Pushes a TLB invalidation job for execution, using @fence as a dependency. + * Storage for @fence must be preallocated with xe_tlb_inval_job_alloc_dep * prior to this call if @fence is not signaled. Takes a reference to the job’s * finished fence, which the caller is responsible for releasing, and return it * to the caller. This function is safe to be called in the path of reclaim. * * Return: Job's finished fence on success, cannot fail */ -struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, - struct xe_migrate *m, - struct dma_fence *fence) +struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence) { - struct xe_gt_tlb_inval_fence *ifence = + struct xe_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); if (!dma_fence_is_signaled(fence)) { @@ -202,20 +196,20 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, /* * Can be in path of reclaim, hence the preallocation of fence - * storage in xe_gt_tlb_inval_job_alloc_dep. Verify caller did + * storage in xe_tlb_inval_job_alloc_dep. Verify caller did * this correctly. */ - xe_assert(gt_to_xe(job->gt), + xe_assert(gt_to_xe(job->q->gt), xa_load(&job->dep.drm.dependencies, 0) == dma_fence_get_stub()); dma_fence_get(fence); /* ref released once dependency processed by scheduler */ ptr = xa_store(&job->dep.drm.dependencies, 0, fence, GFP_ATOMIC); - xe_assert(gt_to_xe(job->gt), !xa_is_err(ptr)); + xe_assert(gt_to_xe(job->q->gt), !xa_is_err(ptr)); } - xe_gt_tlb_inval_job_get(job); /* Pairs with put in free_job */ + xe_tlb_inval_job_get(job); /* Pairs with put in free_job */ job->fence_armed = true; /* @@ -225,8 +219,8 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, */ xe_migrate_job_lock(m, job->q); - /* Creation ref pairs with put in xe_gt_tlb_inval_job_destroy */ - xe_gt_tlb_inval_fence_init(job->gt, ifence, false); + /* Creation ref pairs with put in xe_tlb_inval_job_destroy */ + xe_tlb_inval_fence_init(job->tlb_inval, ifence, false); dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */ drm_sched_job_arm(&job->dep.drm); @@ -241,7 +235,7 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, /* * Not using job->fence, as it has its own dma-fence context, which does - * not allow GT TLB invalidation fences on the same queue, GT tuple to + * not allow TLB invalidation fences on the same queue, GT tuple to * be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler * context and job's finished fence, which enables squashing. */ @@ -249,26 +243,26 @@ struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job, } /** - * xe_gt_tlb_inval_job_get() - Get a reference to GT TLB invalidation job - * @job: GT TLB invalidation job object + * xe_tlb_inval_job_get() - Get a reference to TLB invalidation job + * @job: TLB invalidation job object * - * Increment the GT TLB invalidation job's reference count + * Increment the TLB invalidation job's reference count */ -void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job) +void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job) { kref_get(&job->refcount); } /** - * xe_gt_tlb_inval_job_put() - Put a reference to GT TLB invalidation job - * @job: GT TLB invalidation job object + * xe_tlb_inval_job_put() - Put a reference to TLB invalidation job + * @job: TLB invalidation job object * - * Decrement the GT TLB invalidation job's reference count, call - * xe_gt_tlb_inval_job_destroy when reference count == 0. Skips decrement if + * Decrement the TLB invalidation job's reference count, call + * xe_tlb_inval_job_destroy when reference count == 0. Skips decrement if * input @job is NULL or IS_ERR. */ -void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job) +void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job) { if (!IS_ERR_OR_NULL(job)) - kref_put(&job->refcount, xe_gt_tlb_inval_job_destroy); + kref_put(&job->refcount, xe_tlb_inval_job_destroy); } diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h new file mode 100644 index 000000000000..e63edcb26b50 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_TLB_INVAL_JOB_H_ +#define _XE_TLB_INVAL_JOB_H_ + +#include + +struct dma_fence; +struct xe_dep_scheduler; +struct xe_exec_queue; +struct xe_tlb_inval; +struct xe_tlb_inval_job; +struct xe_migrate; + +struct xe_tlb_inval_job * +xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, + struct xe_dep_scheduler *dep_scheduler, + u64 start, u64 end, u32 asid); + +int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job); + +struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, + struct xe_migrate *m, + struct dma_fence *fence); + +void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job); + +void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h similarity index 56% rename from drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h rename to drivers/gpu/drm/xe/xe_tlb_inval_types.h index 442f72b78ccf..6d14b9f17b91 100644 --- a/drivers/gpu/drm/xe/xe_gt_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -3,58 +3,57 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_GT_TLB_INVAL_TYPES_H_ -#define _XE_GT_TLB_INVAL_TYPES_H_ +#ifndef _XE_TLB_INVAL_TYPES_H_ +#define _XE_TLB_INVAL_TYPES_H_ #include #include -struct xe_gt; - /** struct xe_tlb_inval - TLB invalidation client */ struct xe_tlb_inval { + /** @private: Backend private pointer */ + void *private; /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ #define TLB_INVALIDATION_SEQNO_MAX 0x100000 int seqno; /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ struct mutex seqno_lock; /** - * @tlb_inval.seqno_recv: last received TLB invalidation seqno, - * protected by CT lock + * @seqno_recv: last received TLB invalidation seqno, protected by + * CT lock */ int seqno_recv; /** - * @tlb_inval.pending_fences: list of pending fences waiting TLB - * invaliations, protected by CT lock + * @pending_fences: list of pending fences waiting TLB invaliations, + * protected CT lock */ struct list_head pending_fences; /** - * @tlb_inval.pending_lock: protects @tlb_inval.pending_fences - * and updating @tlb_inval.seqno_recv. + * @pending_lock: protects @pending_fences and updating @seqno_recv. */ spinlock_t pending_lock; /** - * @tlb_inval.fence_tdr: schedules a delayed call to - * xe_gt_tlb_fence_timeout after the timeut interval is over. + * @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after + * the timeout interval is over. */ struct delayed_work fence_tdr; - /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */ + /** @job_wq: schedules TLB invalidation jobs */ struct workqueue_struct *job_wq; /** @tlb_inval.lock: protects TLB invalidation fences */ spinlock_t lock; }; /** - * struct xe_gt_tlb_inval_fence - XE GT TLB invalidation fence + * struct xe_tlb_inval_fence - TLB invalidation fence * - * Optionally passed to xe_gt_tlb_inval and will be signaled upon TLB + * Optionally passed to xe_tlb_inval* functions and will be signaled upon TLB * invalidation completion. */ -struct xe_gt_tlb_inval_fence { +struct xe_tlb_inval_fence { /** @base: dma fence base */ struct dma_fence base; - /** @gt: GT which fence belong to */ - struct xe_gt *gt; + /** @tlb_inval: TLB invalidation client which fence belong to */ + struct xe_tlb_inval *tlb_inval; /** @link: link into list of pending tlb fences */ struct list_head link; /** @seqno: seqno of TLB invalidation to signal fence one */ diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index 36538f50d06f..314f42fcbcbd 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -14,10 +14,10 @@ #include "xe_exec_queue_types.h" #include "xe_gpu_scheduler_types.h" -#include "xe_gt_tlb_inval_types.h" #include "xe_gt_types.h" #include "xe_guc_exec_queue_types.h" #include "xe_sched_job.h" +#include "xe_tlb_inval_types.h" #include "xe_vm.h" #define __dev_name_xe(xe) dev_name((xe)->drm.dev) @@ -25,13 +25,13 @@ #define __dev_name_gt(gt) __dev_name_xe(gt_to_xe((gt))) #define __dev_name_eq(q) __dev_name_gt((q)->gt) -DECLARE_EVENT_CLASS(xe_gt_tlb_inval_fence, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), +DECLARE_EVENT_CLASS(xe_tlb_inval_fence, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), TP_ARGS(xe, fence), TP_STRUCT__entry( __string(dev, __dev_name_xe(xe)) - __field(struct xe_gt_tlb_inval_fence *, fence) + __field(struct xe_tlb_inval_fence *, fence) __field(int, seqno) ), @@ -45,23 +45,23 @@ DECLARE_EVENT_CLASS(xe_gt_tlb_inval_fence, __get_str(dev), __entry->fence, __entry->seqno) ); -DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_send, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_send, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_recv, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_recv, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_signal, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_signal, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); -DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_timeout, - TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence), +DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_timeout, + TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence), TP_ARGS(xe, fence) ); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index b758e9a6837a..3ff3c67aa79d 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -28,7 +28,6 @@ #include "xe_drm_client.h" #include "xe_exec_queue.h" #include "xe_gt_pagefault.h" -#include "xe_gt_tlb_inval.h" #include "xe_migrate.h" #include "xe_pat.h" #include "xe_pm.h" @@ -39,6 +38,7 @@ #include "xe_svm.h" #include "xe_sync.h" #include "xe_tile.h" +#include "xe_tlb_inval.h" #include "xe_trace_bo.h" #include "xe_wa.h" #include "xe_hmm.h" @@ -1898,7 +1898,7 @@ static void xe_vm_close(struct xe_vm *vm) xe_pt_clear(xe, vm->pt_root[id]); for_each_gt(gt, xe, id) - xe_gt_tlb_inval_vm(gt, vm); + xe_tlb_inval_vm(>->tlb_inval, vm); } } @@ -4046,7 +4046,7 @@ void xe_vm_unlock(struct xe_vm *vm) int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, u64 end, u8 tile_mask) { - struct xe_gt_tlb_inval_fence + struct xe_tlb_inval_fence fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE]; struct xe_tile *tile; u32 fence_id = 0; @@ -4060,11 +4060,12 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, if (!(tile_mask & BIT(id))) continue; - xe_gt_tlb_inval_fence_init(tile->primary_gt, - &fence[fence_id], true); + xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval, + &fence[fence_id], true); - err = xe_gt_tlb_inval_range(tile->primary_gt, &fence[fence_id], - start, end, vm->usm.asid); + err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval, + &fence[fence_id], start, end, + vm->usm.asid); if (err) goto wait; ++fence_id; @@ -4072,11 +4073,12 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, if (!tile->media_gt) continue; - xe_gt_tlb_inval_fence_init(tile->media_gt, - &fence[fence_id], true); + xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval, + &fence[fence_id], true); - err = xe_gt_tlb_inval_range(tile->media_gt, &fence[fence_id], - start, end, vm->usm.asid); + err = xe_tlb_inval_range(&tile->media_gt->tlb_inval, + &fence[fence_id], start, end, + vm->usm.asid); if (err) goto wait; ++fence_id; @@ -4084,7 +4086,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, wait: for (id = 0; id < fence_id; ++id) - xe_gt_tlb_inval_fence_wait(&fence[id]); + xe_tlb_inval_fence_wait(&fence[id]); return err; } From 9aff63cf3791e464e4762ca79bd5d5b9643edce6 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:09 +0000 Subject: [PATCH 162/169] drm/xe: Prep TLB invalidation fence before sending It is a bit backwards to add a TLB invalidation fence to the pending list after issuing the invalidation. Perform this step before issuing the TLB invalidation in a helper function. v2: Make sure the seqno_lock mutex covers the send as well (Matt) Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-8-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_tlb_inval.c | 109 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index f4b7c0c74894..dff26f7954ec 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -66,19 +66,19 @@ __inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) static void inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) { + lockdep_assert_held(&fence->tlb_inval->pending_lock); + list_del(&fence->link); __inval_fence_signal(xe, fence); } -void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence) +static void +inval_fence_signal_unlocked(struct xe_device *xe, + struct xe_tlb_inval_fence *fence) { - struct xe_gt *gt; - - if (WARN_ON_ONCE(!fence->tlb_inval)) - return; - - gt = fence->tlb_inval->private; - __inval_fence_signal(gt_to_xe(gt), fence); + spin_lock_irq(&fence->tlb_inval->pending_lock); + inval_fence_signal(xe, fence); + spin_unlock_irq(&fence->tlb_inval->pending_lock); } static void xe_gt_tlb_fence_timeout(struct work_struct *work) @@ -221,14 +221,10 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) return seqno_recv >= seqno; } -static int send_tlb_inval(struct xe_guc *guc, - struct xe_tlb_inval_fence *fence, +static int send_tlb_inval(struct xe_guc *guc, struct xe_tlb_inval_fence *fence, u32 *action, int len) { struct xe_gt *gt = guc_to_gt(guc); - struct xe_device *xe = gt_to_xe(gt); - int seqno; - int ret; xe_gt_assert(gt, fence); @@ -238,47 +234,36 @@ static int send_tlb_inval(struct xe_guc *guc, * need to be updated. */ - mutex_lock(>->tlb_inval.seqno_lock); - seqno = gt->tlb_inval.seqno; - fence->seqno = seqno; - trace_xe_tlb_inval_fence_send(xe, fence); - action[1] = seqno; - ret = xe_guc_ct_send(&guc->ct, action, len, - G2H_LEN_DW_TLB_INVALIDATE, 1); - if (!ret) { - spin_lock_irq(>->tlb_inval.pending_lock); - /* - * We haven't actually published the TLB fence as per - * pending_fences, but in theory our seqno could have already - * been written as we acquired the pending_lock. In such a case - * we can just go ahead and signal the fence here. - */ - if (tlb_inval_seqno_past(gt, seqno)) { - __inval_fence_signal(xe, fence); - } else { - fence->inval_time = ktime_get(); - list_add_tail(&fence->link, - >->tlb_inval.pending_fences); - - if (list_is_singular(>->tlb_inval.pending_fences)) - queue_delayed_work(system_wq, - >->tlb_inval.fence_tdr, - tlb_timeout_jiffies(gt)); - } - spin_unlock_irq(>->tlb_inval.pending_lock); - } else { - __inval_fence_signal(xe, fence); - } - if (!ret) { - gt->tlb_inval.seqno = (gt->tlb_inval.seqno + 1) % - TLB_INVALIDATION_SEQNO_MAX; - if (!gt->tlb_inval.seqno) - gt->tlb_inval.seqno = 1; - } - mutex_unlock(>->tlb_inval.seqno_lock); xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); + action[1] = fence->seqno; - return ret; + return xe_guc_ct_send(&guc->ct, action, len, + G2H_LEN_DW_TLB_INVALIDATE, 1); +} + +static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) +{ + struct xe_tlb_inval *tlb_inval = fence->tlb_inval; + struct xe_gt *gt = tlb_inval->private; + struct xe_device *xe = gt_to_xe(gt); + + fence->seqno = tlb_inval->seqno; + trace_xe_tlb_inval_fence_send(xe, fence); + + spin_lock_irq(&tlb_inval->pending_lock); + fence->inval_time = ktime_get(); + list_add_tail(&fence->link, &tlb_inval->pending_fences); + + if (list_is_singular(&tlb_inval->pending_fences)) + queue_delayed_work(system_wq, + &tlb_inval->fence_tdr, + tlb_timeout_jiffies(gt)); + spin_unlock_irq(&tlb_inval->pending_lock); + + tlb_inval->seqno = (tlb_inval->seqno + 1) % + TLB_INVALIDATION_SEQNO_MAX; + if (!tlb_inval->seqno) + tlb_inval->seqno = 1; } #define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ @@ -306,7 +291,14 @@ static int xe_tlb_inval_guc(struct xe_gt *gt, }; int ret; + mutex_lock(>->tlb_inval.seqno_lock); + xe_tlb_inval_fence_prep(fence); + ret = send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); + if (ret < 0) + inval_fence_signal_unlocked(gt_to_xe(gt), fence); + mutex_unlock(>->tlb_inval.seqno_lock); + /* * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches * should be nuked on a GT reset so this error can be ignored. @@ -433,7 +425,7 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, #define MAX_TLB_INVALIDATION_LEN 7 u32 action[MAX_TLB_INVALIDATION_LEN]; u64 length = end - start; - int len = 0; + int len = 0, ret; xe_gt_assert(gt, fence); @@ -494,7 +486,16 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); - return send_tlb_inval(>->uc.guc, fence, action, len); + mutex_lock(>->tlb_inval.seqno_lock); + xe_tlb_inval_fence_prep(fence); + + ret = send_tlb_inval(>->uc.guc, fence, action, + ARRAY_SIZE(action)); + if (ret < 0) + inval_fence_signal_unlocked(xe, fence); + mutex_unlock(>->tlb_inval.seqno_lock); + + return ret; } /** From 8443e8c448cfe1082f2bbad41fe6e56e8d3b5bba Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:10 +0000 Subject: [PATCH 163/169] drm/xe: Add helpers to send TLB invalidations Break out the GuC specific code into helpers as part of the process to decouple frontback TLB invalidation code from the backend. Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-9-stuart.summers@intel.com --- drivers/gpu/drm/xe/xe_tlb_inval.c | 234 +++++++++++++++--------------- 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index dff26f7954ec..d81a45f1dd7c 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -221,12 +221,11 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) return seqno_recv >= seqno; } -static int send_tlb_inval(struct xe_guc *guc, struct xe_tlb_inval_fence *fence, - u32 *action, int len) +static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len) { struct xe_gt *gt = guc_to_gt(guc); - xe_gt_assert(gt, fence); + xe_gt_assert(gt, action[1]); /* Seqno */ /* * XXX: The seqno algorithm relies on TLB invalidation being processed @@ -235,7 +234,6 @@ static int send_tlb_inval(struct xe_guc *guc, struct xe_tlb_inval_fence *fence, */ xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); - action[1] = fence->seqno; return xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_TLB_INVALIDATE, 1); @@ -270,91 +268,15 @@ static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ XE_GUC_TLB_INVAL_FLUSH_CACHE) -/** - * xe_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC - * @gt: GT structure - * @fence: invalidation fence which will be signal on TLB invalidation - * completion - * - * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and - * caller can use the invalidation fence to wait for completion. - * - * Return: 0 on success, negative error code on error - */ -static int xe_tlb_inval_guc(struct xe_gt *gt, - struct xe_tlb_inval_fence *fence) +static int send_tlb_inval_ggtt(struct xe_gt *gt, int seqno) { u32 action[] = { XE_GUC_ACTION_TLB_INVALIDATION, - 0, /* seqno, replaced in send_tlb_inval */ + seqno, MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), }; - int ret; - mutex_lock(>->tlb_inval.seqno_lock); - xe_tlb_inval_fence_prep(fence); - - ret = send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); - if (ret < 0) - inval_fence_signal_unlocked(gt_to_xe(gt), fence); - mutex_unlock(>->tlb_inval.seqno_lock); - - /* - * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches - * should be nuked on a GT reset so this error can be ignored. - */ - if (ret == -ECANCELED) - return 0; - - return ret; -} - -/** - * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT - * @tlb_inval: TLB invalidation client - * - * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is - * synchronous. - * - * Return: 0 on success, negative error code on error - */ -int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) -{ - struct xe_gt *gt = tlb_inval->private; - struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; - - if (xe_guc_ct_enabled(>->uc.guc.ct) && - gt->uc.guc.submission_state.enabled) { - struct xe_tlb_inval_fence fence; - int ret; - - xe_tlb_inval_fence_init(tlb_inval, &fence, true); - ret = xe_tlb_inval_guc(gt, &fence); - if (ret) - return ret; - - xe_tlb_inval_fence_wait(&fence); - } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { - struct xe_mmio *mmio = >->mmio; - - if (IS_SRIOV_VF(xe)) - return 0; - - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, - PVC_GUC_TLB_INV_DESC1_INVALIDATE); - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0, - PVC_GUC_TLB_INV_DESC0_VALID); - } else { - xe_mmio_write32(mmio, GUC_TLB_INV_CR, - GUC_TLB_INV_CR_INVALIDATE); - } - xe_force_wake_put(gt_to_fw(gt), fw_ref); - } - - return 0; + return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action)); } static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, @@ -369,7 +291,7 @@ static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, xe_gt_assert(gt, fence); - return send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action)); + return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action)); } /** @@ -401,43 +323,17 @@ int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, */ #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) -/** - * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range - * @tlb_inval: TLB invalidation client - * @fence: invalidation fence which will be signal on TLB invalidation - * completion - * @start: start address - * @end: end address - * @asid: address space id - * - * Issue a range based TLB invalidation if supported, if not fallback to a full - * TLB invalidation. Completion of TLB is asynchronous and caller can use - * the invalidation fence to wait for completion. - * - * Return: Negative error code on error, 0 on success - */ -int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, - struct xe_tlb_inval_fence *fence, u64 start, u64 end, - u32 asid) +static int send_tlb_inval_ppgtt(struct xe_gt *gt, u64 start, u64 end, + u32 asid, int seqno) { - struct xe_gt *gt = tlb_inval->private; - struct xe_device *xe = gt_to_xe(gt); #define MAX_TLB_INVALIDATION_LEN 7 u32 action[MAX_TLB_INVALIDATION_LEN]; u64 length = end - start; - int len = 0, ret; - - xe_gt_assert(gt, fence); - - /* Execlists not supported */ - if (gt_to_xe(gt)->info.force_execlist) { - __inval_fence_signal(xe, fence); - return 0; - } + int len = 0; action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; - action[len++] = 0; /* seqno, replaced in send_tlb_inval */ - if (!xe->info.has_range_tlb_inval || + action[len++] = seqno; + if (!gt_to_xe(gt)->info.has_range_tlb_inval || length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); } else { @@ -486,11 +382,115 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); + return send_tlb_inval(>->uc.guc, action, len); +} + +static int __xe_tlb_inval_ggtt(struct xe_gt *gt, + struct xe_tlb_inval_fence *fence) +{ + int ret; + mutex_lock(>->tlb_inval.seqno_lock); xe_tlb_inval_fence_prep(fence); - ret = send_tlb_inval(>->uc.guc, fence, action, - ARRAY_SIZE(action)); + ret = send_tlb_inval_ggtt(gt, fence->seqno); + if (ret < 0) + inval_fence_signal_unlocked(gt_to_xe(gt), fence); + mutex_unlock(>->tlb_inval.seqno_lock); + + /* + * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches + * should be nuked on a GT reset so this error can be ignored. + */ + if (ret == -ECANCELED) + return 0; + + return ret; +} + +/** + * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT + * @tlb_inval: TLB invalidation client + * + * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is + * synchronous. + * + * Return: 0 on success, negative error code on error + */ +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) +{ + struct xe_gt *gt = tlb_inval->private; + struct xe_device *xe = gt_to_xe(gt); + unsigned int fw_ref; + + if (xe_guc_ct_enabled(>->uc.guc.ct) && + gt->uc.guc.submission_state.enabled) { + struct xe_tlb_inval_fence fence; + int ret; + + xe_tlb_inval_fence_init(tlb_inval, &fence, true); + ret = __xe_tlb_inval_ggtt(gt, &fence); + if (ret) + return ret; + + xe_tlb_inval_fence_wait(&fence); + } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { + struct xe_mmio *mmio = >->mmio; + + if (IS_SRIOV_VF(xe)) + return 0; + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, + PVC_GUC_TLB_INV_DESC1_INVALIDATE); + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0, + PVC_GUC_TLB_INV_DESC0_VALID); + } else { + xe_mmio_write32(mmio, GUC_TLB_INV_CR, + GUC_TLB_INV_CR_INVALIDATE); + } + xe_force_wake_put(gt_to_fw(gt), fw_ref); + } + + return 0; +} + +/** + * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range + * @tlb_inval: TLB invalidation client + * @fence: invalidation fence which will be signal on TLB invalidation + * completion + * @start: start address + * @end: end address + * @asid: address space id + * + * Issue a range based TLB invalidation if supported, if not fallback to a full + * TLB invalidation. Completion of TLB is asynchronous and caller can use + * the invalidation fence to wait for completion. + * + * Return: Negative error code on error, 0 on success + */ +int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, + struct xe_tlb_inval_fence *fence, u64 start, u64 end, + u32 asid) +{ + struct xe_gt *gt = tlb_inval->private; + struct xe_device *xe = gt_to_xe(gt); + int ret; + + xe_gt_assert(gt, fence); + + /* Execlists not supported */ + if (xe->info.force_execlist) { + __inval_fence_signal(xe, fence); + return 0; + } + + mutex_lock(>->tlb_inval.seqno_lock); + xe_tlb_inval_fence_prep(fence); + + ret = send_tlb_inval_ppgtt(gt, start, end, asid, fence->seqno); if (ret < 0) inval_fence_signal_unlocked(xe, fence); mutex_unlock(>->tlb_inval.seqno_lock); From db16f9d90c1d97a2b5483b4b856625875bffe860 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 26 Aug 2025 18:29:11 +0000 Subject: [PATCH 164/169] drm/xe: Split TLB invalidation code in frontend and backend The frontend exposes an API to the driver to send invalidations, handles sequence number assignment, synchronization (fences), and provides a timeout mechanism. The backend issues the actual invalidation to the hardware (or firmware). The new layering easily allows issuing TLB invalidations to different hardware or firmware interfaces. Normalize some naming while here too. Signed-off-by: Stuart Summers Reviewed-by: Stuart Summers Signed-off-by: Matthew Brost Link: https://lore.kernel.org/r/20250826182911.392550-10-stuart.summers@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_gt.c | 2 - drivers/gpu/drm/xe/xe_guc_ct.c | 2 +- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 242 +++++++++++ drivers/gpu/drm/xe/xe_guc_tlb_inval.h | 19 + drivers/gpu/drm/xe/xe_tlb_inval.c | 537 +++++++----------------- drivers/gpu/drm/xe/xe_tlb_inval.h | 15 +- drivers/gpu/drm/xe/xe_tlb_inval_types.h | 67 ++- 8 files changed, 500 insertions(+), 385 deletions(-) create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.c create mode 100644 drivers/gpu/drm/xe/xe_guc_tlb_inval.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 2b67debb8bc4..987e4fe10538 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -75,6 +75,7 @@ xe-y += xe_bb.o \ xe_guc_log.o \ xe_guc_pc.o \ xe_guc_submit.o \ + xe_guc_tlb_inval.o \ xe_heci_gsc.o \ xe_huc.o \ xe_hw_engine.o \ diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 67ee7cdbd6ec..34505a6d93ed 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -603,8 +603,6 @@ static void xe_gt_fini(void *arg) struct xe_gt *gt = arg; int i; - xe_gt_tlb_inval_fini(gt); - for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) xe_hw_fence_irq_finish(>->fence_irq[i]); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 5f38041cff4c..848065a25c44 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -30,9 +30,9 @@ #include "xe_guc_log.h" #include "xe_guc_relay.h" #include "xe_guc_submit.h" +#include "xe_guc_tlb_inval.h" #include "xe_map.h" #include "xe_pm.h" -#include "xe_tlb_inval.h" #include "xe_trace_guc.h" static void receive_g2h(struct xe_guc_ct *ct); diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c new file mode 100644 index 000000000000..6bf2103602f8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "abi/guc_actions_abi.h" + +#include "xe_device.h" +#include "xe_gt_stats.h" +#include "xe_gt_types.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_tlb_inval.h" +#include "xe_force_wake.h" +#include "xe_mmio.h" +#include "xe_tlb_inval.h" + +#include "regs/xe_guc_regs.h" + +/* + * XXX: The seqno algorithm relies on TLB invalidation being processed in order + * which they currently are by the GuC, if that changes the algorithm will need + * to be updated. + */ + +static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len) +{ + struct xe_gt *gt = guc_to_gt(guc); + + xe_gt_assert(gt, action[1]); /* Seqno */ + + xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); + return xe_guc_ct_send(&guc->ct, action, len, + G2H_LEN_DW_TLB_INVALIDATE, 1); +} + +#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ + XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ + XE_GUC_TLB_INVAL_FLUSH_CACHE) + +static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno) +{ + struct xe_guc *guc = tlb_inval->private; + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION_ALL, + seqno, + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), + }; + + return send_tlb_inval(guc, action, ARRAY_SIZE(action)); +} + +static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) +{ + struct xe_guc *guc = tlb_inval->private; + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + + /* + * Returning -ECANCELED in this function is squashed at the caller and + * signals waiters. + */ + + if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) { + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION, + seqno, + MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), + }; + + return send_tlb_inval(guc, action, ARRAY_SIZE(action)); + } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { + struct xe_mmio *mmio = >->mmio; + unsigned int fw_ref; + + if (IS_SRIOV_VF(xe)) + return -ECANCELED; + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, + PVC_GUC_TLB_INV_DESC1_INVALIDATE); + xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0, + PVC_GUC_TLB_INV_DESC0_VALID); + } else { + xe_mmio_write32(mmio, GUC_TLB_INV_CR, + GUC_TLB_INV_CR_INVALIDATE); + } + xe_force_wake_put(gt_to_fw(gt), fw_ref); + } + + return -ECANCELED; +} + +/* + * Ensure that roundup_pow_of_two(length) doesn't overflow. + * Note that roundup_pow_of_two() operates on unsigned long, + * not on u64. + */ +#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) + +static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, + u64 start, u64 end, u32 asid) +{ +#define MAX_TLB_INVALIDATION_LEN 7 + struct xe_guc *guc = tlb_inval->private; + struct xe_gt *gt = guc_to_gt(guc); + u32 action[MAX_TLB_INVALIDATION_LEN]; + u64 length = end - start; + int len = 0; + + if (guc_to_xe(guc)->info.force_execlist) + return -ECANCELED; + + action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; + action[len++] = seqno; + if (!gt_to_xe(gt)->info.has_range_tlb_inval || + length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); + } else { + u64 orig_start = start; + u64 align; + + if (length < SZ_4K) + length = SZ_4K; + + /* + * We need to invalidate a higher granularity if start address + * is not aligned to length. When start is not aligned with + * length we need to find the length large enough to create an + * address mask covering the required range. + */ + align = roundup_pow_of_two(length); + start = ALIGN_DOWN(start, align); + end = ALIGN(end, align); + length = align; + while (start + length < end) { + length <<= 1; + start = ALIGN_DOWN(orig_start, length); + } + + /* + * Minimum invalidation size for a 2MB page that the hardware + * expects is 16MB + */ + if (length >= SZ_2M) { + length = max_t(u64, SZ_16M, length); + start = ALIGN_DOWN(orig_start, length); + } + + xe_gt_assert(gt, length >= SZ_4K); + xe_gt_assert(gt, is_power_of_2(length)); + xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, + ilog2(SZ_2M) + 1))); + xe_gt_assert(gt, IS_ALIGNED(start, length)); + + action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE); + action[len++] = asid; + action[len++] = lower_32_bits(start); + action[len++] = upper_32_bits(start); + action[len++] = ilog2(length) - ilog2(SZ_4K); + } + + xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); + + return send_tlb_inval(guc, action, len); +} + +static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + return xe_guc_ct_initialized(&guc->ct); +} + +static void tlb_inval_flush(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + LNL_FLUSH_WORK(&guc->ct.g2h_worker); +} + +static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval) +{ + struct xe_guc *guc = tlb_inval->private; + + /* this reflects what HW/GuC needs to process TLB inv request */ + const long hw_tlb_timeout = HZ / 4; + + /* this estimates actual delay caused by the CTB transport */ + long delay = xe_guc_ct_queue_proc_time_jiffies(&guc->ct); + + return hw_tlb_timeout + 2 * delay; +} + +static const struct xe_tlb_inval_ops guc_tlb_inval_ops = { + .all = send_tlb_inval_all, + .ggtt = send_tlb_inval_ggtt, + .ppgtt = send_tlb_inval_ppgtt, + .initialized = tlb_inval_initialized, + .flush = tlb_inval_flush, + .timeout_delay = tlb_inval_timeout_delay, +}; + +/** + * xe_guc_tlb_inval_init_early() - Init GuC TLB invalidation early + * @guc: GuC object + * @tlb_inval: TLB invalidation client + * + * Inititialize GuC TLB invalidation by setting back pointer in TLB invalidation + * client to the GuC and setting GuC backend ops. + */ +void xe_guc_tlb_inval_init_early(struct xe_guc *guc, + struct xe_tlb_inval *tlb_inval) +{ + tlb_inval->private = guc; + tlb_inval->ops = &guc_tlb_inval_ops; +} + +/** + * xe_guc_tlb_inval_done_handler() - TLB invalidation done handler + * @guc: guc + * @msg: message indicating TLB invalidation done + * @len: length of message + * + * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any + * invalidation fences for seqno. Algorithm for this depends on seqno being + * received in-order and asserts this assumption. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + + if (unlikely(len != 1)) + return -EPROTO; + + xe_tlb_inval_done_handler(>->tlb_inval, msg[0]); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.h b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h new file mode 100644 index 000000000000..07d668b02e3d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_GUC_TLB_INVAL_H_ +#define _XE_GUC_TLB_INVAL_H_ + +#include + +struct xe_guc; +struct xe_tlb_inval; + +void xe_guc_tlb_inval_init_early(struct xe_guc *guc, + struct xe_tlb_inval *tlb_inval); + +int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index d81a45f1dd7c..e6e97b5a7b5c 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -12,51 +12,45 @@ #include "xe_gt_printk.h" #include "xe_guc.h" #include "xe_guc_ct.h" +#include "xe_guc_tlb_inval.h" #include "xe_gt_stats.h" #include "xe_tlb_inval.h" #include "xe_mmio.h" #include "xe_pm.h" -#include "xe_sriov.h" +#include "xe_tlb_inval.h" #include "xe_trace.h" -#include "regs/xe_guc_regs.h" + +/** + * DOC: Xe TLB invalidation + * + * Xe TLB invalidation is implemented in two layers. The first is the frontend + * API, which provides an interface for TLB invalidations to the driver code. + * The frontend handles seqno assignment, synchronization (fences), and the + * timeout mechanism. The frontend is implemented via an embedded structure + * xe_tlb_inval that includes a set of ops hooking into the backend. The backend + * interacts with the hardware (or firmware) to perform the actual invalidation. + */ #define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS -/* - * TLB inval depends on pending commands in the CT queue and then the real - * invalidation time. Double up the time to process full CT queue - * just to be on the safe side. - */ -static long tlb_timeout_jiffies(struct xe_gt *gt) -{ - /* this reflects what HW/GuC needs to process TLB inv request */ - const long hw_tlb_timeout = HZ / 4; - - /* this estimates actual delay caused by the CTB transport */ - long delay = xe_guc_ct_queue_proc_time_jiffies(>->uc.guc.ct); - - return hw_tlb_timeout + 2 * delay; -} - static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence) { - struct xe_gt *gt; - if (WARN_ON_ONCE(!fence->tlb_inval)) return; - gt = fence->tlb_inval->private; - - xe_pm_runtime_put(gt_to_xe(gt)); + xe_pm_runtime_put(fence->tlb_inval->xe); fence->tlb_inval = NULL; /* fini() should be called once */ } static void -__inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) +xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence) { bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags); - trace_xe_tlb_inval_fence_signal(xe, fence); + lockdep_assert_held(&fence->tlb_inval->pending_lock); + + list_del(&fence->link); + trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence); xe_tlb_inval_fence_fini(fence); dma_fence_signal(&fence->base); if (!stack) @@ -64,57 +58,65 @@ __inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) } static void -inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence) +xe_tlb_inval_fence_signal_unlocked(struct xe_tlb_inval_fence *fence) { - lockdep_assert_held(&fence->tlb_inval->pending_lock); + struct xe_tlb_inval *tlb_inval = fence->tlb_inval; - list_del(&fence->link); - __inval_fence_signal(xe, fence); + spin_lock_irq(&tlb_inval->pending_lock); + xe_tlb_inval_fence_signal(fence); + spin_unlock_irq(&tlb_inval->pending_lock); } -static void -inval_fence_signal_unlocked(struct xe_device *xe, - struct xe_tlb_inval_fence *fence) +static void xe_tlb_inval_fence_timeout(struct work_struct *work) { - spin_lock_irq(&fence->tlb_inval->pending_lock); - inval_fence_signal(xe, fence); - spin_unlock_irq(&fence->tlb_inval->pending_lock); -} - -static void xe_gt_tlb_fence_timeout(struct work_struct *work) -{ - struct xe_gt *gt = container_of(work, struct xe_gt, - tlb_inval.fence_tdr.work); - struct xe_device *xe = gt_to_xe(gt); + struct xe_tlb_inval *tlb_inval = container_of(work, struct xe_tlb_inval, + fence_tdr.work); + struct xe_device *xe = tlb_inval->xe; struct xe_tlb_inval_fence *fence, *next; + long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval); - LNL_FLUSH_WORK(>->uc.guc.ct.g2h_worker); + tlb_inval->ops->flush(tlb_inval); - spin_lock_irq(>->tlb_inval.pending_lock); + spin_lock_irq(&tlb_inval->pending_lock); list_for_each_entry_safe(fence, next, - >->tlb_inval.pending_fences, link) { + &tlb_inval->pending_fences, link) { s64 since_inval_ms = ktime_ms_delta(ktime_get(), fence->inval_time); - if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt)) + if (msecs_to_jiffies(since_inval_ms) < timeout_delay) break; trace_xe_tlb_inval_fence_timeout(xe, fence); - xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d", - fence->seqno, gt->tlb_inval.seqno_recv); + drm_err(&xe->drm, + "TLB invalidation fence timeout, seqno=%d recv=%d", + fence->seqno, tlb_inval->seqno_recv); fence->base.error = -ETIME; - inval_fence_signal(xe, fence); + xe_tlb_inval_fence_signal(fence); } - if (!list_empty(>->tlb_inval.pending_fences)) - queue_delayed_work(system_wq, - >->tlb_inval.fence_tdr, - tlb_timeout_jiffies(gt)); - spin_unlock_irq(>->tlb_inval.pending_lock); + if (!list_empty(&tlb_inval->pending_fences)) + queue_delayed_work(system_wq, &tlb_inval->fence_tdr, + timeout_delay); + spin_unlock_irq(&tlb_inval->pending_lock); } /** - * xe_gt_tlb_inval_init_early - Initialize GT TLB invalidation state + * tlb_inval_fini - Clean up TLB invalidation state + * @drm: @drm_device + * @arg: pointer to struct @xe_tlb_inval + * + * Cancel pending fence workers and clean up any additional + * TLB invalidation state. + */ +static void tlb_inval_fini(struct drm_device *drm, void *arg) +{ + struct xe_tlb_inval *tlb_inval = arg; + + xe_tlb_inval_reset(tlb_inval); +} + +/** + * xe_gt_tlb_inval_init - Initialize TLB invalidation state * @gt: GT structure * * Initialize TLB invalidation state, purely software initialization, should @@ -125,92 +127,84 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) int xe_gt_tlb_inval_init_early(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); + struct xe_tlb_inval *tlb_inval = >->tlb_inval; int err; - gt->tlb_inval.private = gt; - gt->tlb_inval.seqno = 1; - INIT_LIST_HEAD(>->tlb_inval.pending_fences); - spin_lock_init(>->tlb_inval.pending_lock); - spin_lock_init(>->tlb_inval.lock); - INIT_DELAYED_WORK(>->tlb_inval.fence_tdr, - xe_gt_tlb_fence_timeout); + tlb_inval->xe = xe; + tlb_inval->seqno = 1; + INIT_LIST_HEAD(&tlb_inval->pending_fences); + spin_lock_init(&tlb_inval->pending_lock); + spin_lock_init(&tlb_inval->lock); + INIT_DELAYED_WORK(&tlb_inval->fence_tdr, xe_tlb_inval_fence_timeout); - err = drmm_mutex_init(&xe->drm, >->tlb_inval.seqno_lock); + err = drmm_mutex_init(&xe->drm, &tlb_inval->seqno_lock); if (err) return err; - gt->tlb_inval.job_wq = - drmm_alloc_ordered_workqueue(>_to_xe(gt)->drm, "gt-tbl-inval-job-wq", - WQ_MEM_RECLAIM); - if (IS_ERR(gt->tlb_inval.job_wq)) - return PTR_ERR(gt->tlb_inval.job_wq); + tlb_inval->job_wq = drmm_alloc_ordered_workqueue(&xe->drm, + "gt-tbl-inval-job-wq", + WQ_MEM_RECLAIM); + if (IS_ERR(tlb_inval->job_wq)) + return PTR_ERR(tlb_inval->job_wq); - return 0; + /* XXX: Blindly setting up backend to GuC */ + xe_guc_tlb_inval_init_early(>->uc.guc, tlb_inval); + + return drmm_add_action_or_reset(&xe->drm, tlb_inval_fini, tlb_inval); } /** - * xe_tlb_inval_reset - Initialize TLB invalidation reset + * xe_tlb_inval_reset() - TLB invalidation reset * @tlb_inval: TLB invalidation client * * Signal any pending invalidation fences, should be called during a GT reset */ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) { - struct xe_gt *gt = tlb_inval->private; struct xe_tlb_inval_fence *fence, *next; int pending_seqno; /* - * we can get here before the CTs are even initialized if we're wedging - * very early, in which case there are not going to be any pending - * fences so we can bail immediately. + * we can get here before the backends are even initialized if we're + * wedging very early, in which case there are not going to be any + * pendind fences so we can bail immediately. */ - if (!xe_guc_ct_initialized(>->uc.guc.ct)) + if (!tlb_inval->ops->initialized(tlb_inval)) return; /* - * CT channel is already disabled at this point. No new TLB requests can + * Backend is already disabled at this point. No new TLB requests can * appear. */ - mutex_lock(>->tlb_inval.seqno_lock); - spin_lock_irq(>->tlb_inval.pending_lock); - cancel_delayed_work(>->tlb_inval.fence_tdr); + mutex_lock(&tlb_inval->seqno_lock); + spin_lock_irq(&tlb_inval->pending_lock); + cancel_delayed_work(&tlb_inval->fence_tdr); /* * We might have various kworkers waiting for TLB flushes to complete * which are not tracked with an explicit TLB fence, however at this - * stage that will never happen since the CT is already disabled, so - * make sure we signal them here under the assumption that we have + * stage that will never happen since the backend is already disabled, + * so make sure we signal them here under the assumption that we have * completed a full GT reset. */ - if (gt->tlb_inval.seqno == 1) + if (tlb_inval->seqno == 1) pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1; else - pending_seqno = gt->tlb_inval.seqno - 1; - WRITE_ONCE(gt->tlb_inval.seqno_recv, pending_seqno); + pending_seqno = tlb_inval->seqno - 1; + WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno); list_for_each_entry_safe(fence, next, - >->tlb_inval.pending_fences, link) - inval_fence_signal(gt_to_xe(gt), fence); - spin_unlock_irq(>->tlb_inval.pending_lock); - mutex_unlock(>->tlb_inval.seqno_lock); + &tlb_inval->pending_fences, link) + xe_tlb_inval_fence_signal(fence); + spin_unlock_irq(&tlb_inval->pending_lock); + mutex_unlock(&tlb_inval->seqno_lock); } -/** - * - * xe_gt_tlb_inval_fini - Clean up GT TLB invalidation state - * - * Cancel pending fence workers and clean up any additional - * GT TLB invalidation state. - */ -void xe_gt_tlb_inval_fini(struct xe_gt *gt) +static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno) { - xe_gt_tlb_inval_reset(gt); -} + int seqno_recv = READ_ONCE(tlb_inval->seqno_recv); -static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) -{ - int seqno_recv = READ_ONCE(gt->tlb_inval.seqno_recv); + lockdep_assert_held(&tlb_inval->pending_lock); if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2)) return false; @@ -221,41 +215,20 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno) return seqno_recv >= seqno; } -static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len) -{ - struct xe_gt *gt = guc_to_gt(guc); - - xe_gt_assert(gt, action[1]); /* Seqno */ - - /* - * XXX: The seqno algorithm relies on TLB invalidation being processed - * in order which they currently are, if that changes the algorithm will - * need to be updated. - */ - - xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1); - - return xe_guc_ct_send(&guc->ct, action, len, - G2H_LEN_DW_TLB_INVALIDATE, 1); -} - static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) { struct xe_tlb_inval *tlb_inval = fence->tlb_inval; - struct xe_gt *gt = tlb_inval->private; - struct xe_device *xe = gt_to_xe(gt); fence->seqno = tlb_inval->seqno; - trace_xe_tlb_inval_fence_send(xe, fence); + trace_xe_tlb_inval_fence_send(tlb_inval->xe, fence); spin_lock_irq(&tlb_inval->pending_lock); fence->inval_time = ktime_get(); list_add_tail(&fence->link, &tlb_inval->pending_fences); if (list_is_singular(&tlb_inval->pending_fences)) - queue_delayed_work(system_wq, - &tlb_inval->fence_tdr, - tlb_timeout_jiffies(gt)); + queue_delayed_work(system_wq, &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); spin_unlock_irq(&tlb_inval->pending_lock); tlb_inval->seqno = (tlb_inval->seqno + 1) % @@ -264,200 +237,63 @@ static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) tlb_inval->seqno = 1; } -#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ - XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ - XE_GUC_TLB_INVAL_FLUSH_CACHE) - -static int send_tlb_inval_ggtt(struct xe_gt *gt, int seqno) -{ - u32 action[] = { - XE_GUC_ACTION_TLB_INVALIDATION, - seqno, - MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), - }; - - return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action)); -} - -static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, - struct xe_tlb_inval_fence *fence) -{ - u32 action[] = { - XE_GUC_ACTION_TLB_INVALIDATION_ALL, - 0, /* seqno, replaced in send_tlb_inval */ - MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL), - }; - struct xe_gt *gt = tlb_inval->private; - - xe_gt_assert(gt, fence); - - return send_tlb_inval(>->uc.guc, action, ARRAY_SIZE(action)); -} +#define xe_tlb_inval_issue(__tlb_inval, __fence, op, args...) \ +({ \ + int __ret; \ + \ + xe_assert((__tlb_inval)->xe, (__tlb_inval)->ops); \ + xe_assert((__tlb_inval)->xe, (__fence)); \ + \ + mutex_lock(&(__tlb_inval)->seqno_lock); \ + xe_tlb_inval_fence_prep((__fence)); \ + __ret = op((__tlb_inval), (__fence)->seqno, ##args); \ + if (__ret < 0) \ + xe_tlb_inval_fence_signal_unlocked((__fence)); \ + mutex_unlock(&(__tlb_inval)->seqno_lock); \ + \ + __ret == -ECANCELED ? 0 : __ret; \ +}) /** - * xe_gt_tlb_invalidation_all - Invalidate all TLBs across PF and all VFs. - * @gt: the &xe_gt structure - * @fence: the &xe_tlb_inval_fence to be signaled on completion + * xe_tlb_inval_all() - Issue a TLB invalidation for all TLBs + * @tlb_inval: TLB invalidation client + * @fence: invalidation fence which will be signal on TLB invalidation + * completion * - * Send a request to invalidate all TLBs across PF and all VFs. + * Issue a TLB invalidation for all TLBs. Completion of TLB is asynchronous and + * caller can use the invalidation fence to wait for completion. * * Return: 0 on success, negative error code on error */ int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence) { - struct xe_gt *gt = tlb_inval->private; - int err; - - err = send_tlb_inval_all(tlb_inval, fence); - if (err) - xe_gt_err(gt, "TLB invalidation request failed (%pe)", ERR_PTR(err)); - - return err; -} - -/* - * Ensure that roundup_pow_of_two(length) doesn't overflow. - * Note that roundup_pow_of_two() operates on unsigned long, - * not on u64. - */ -#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) - -static int send_tlb_inval_ppgtt(struct xe_gt *gt, u64 start, u64 end, - u32 asid, int seqno) -{ -#define MAX_TLB_INVALIDATION_LEN 7 - u32 action[MAX_TLB_INVALIDATION_LEN]; - u64 length = end - start; - int len = 0; - - action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; - action[len++] = seqno; - if (!gt_to_xe(gt)->info.has_range_tlb_inval || - length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { - action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); - } else { - u64 orig_start = start; - u64 align; - - if (length < SZ_4K) - length = SZ_4K; - - /* - * We need to invalidate a higher granularity if start address - * is not aligned to length. When start is not aligned with - * length we need to find the length large enough to create an - * address mask covering the required range. - */ - align = roundup_pow_of_two(length); - start = ALIGN_DOWN(start, align); - end = ALIGN(end, align); - length = align; - while (start + length < end) { - length <<= 1; - start = ALIGN_DOWN(orig_start, length); - } - - /* - * Minimum invalidation size for a 2MB page that the hardware - * expects is 16MB - */ - if (length >= SZ_2M) { - length = max_t(u64, SZ_16M, length); - start = ALIGN_DOWN(orig_start, length); - } - - xe_gt_assert(gt, length >= SZ_4K); - xe_gt_assert(gt, is_power_of_2(length)); - xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, - ilog2(SZ_2M) + 1))); - xe_gt_assert(gt, IS_ALIGNED(start, length)); - - action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE); - action[len++] = asid; - action[len++] = lower_32_bits(start); - action[len++] = upper_32_bits(start); - action[len++] = ilog2(length) - ilog2(SZ_4K); - } - - xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); - - return send_tlb_inval(>->uc.guc, action, len); -} - -static int __xe_tlb_inval_ggtt(struct xe_gt *gt, - struct xe_tlb_inval_fence *fence) -{ - int ret; - - mutex_lock(>->tlb_inval.seqno_lock); - xe_tlb_inval_fence_prep(fence); - - ret = send_tlb_inval_ggtt(gt, fence->seqno); - if (ret < 0) - inval_fence_signal_unlocked(gt_to_xe(gt), fence); - mutex_unlock(>->tlb_inval.seqno_lock); - - /* - * -ECANCELED indicates the CT is stopped for a GT reset. TLB caches - * should be nuked on a GT reset so this error can be ignored. - */ - if (ret == -ECANCELED) - return 0; - - return ret; + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->all); } /** - * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT + * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT * @tlb_inval: TLB invalidation client * - * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is - * synchronous. + * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and + * caller can use the invalidation fence to wait for completion. * * Return: 0 on success, negative error code on error */ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) { - struct xe_gt *gt = tlb_inval->private; - struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; + struct xe_tlb_inval_fence fence, *fence_ptr = &fence; + int ret; - if (xe_guc_ct_enabled(>->uc.guc.ct) && - gt->uc.guc.submission_state.enabled) { - struct xe_tlb_inval_fence fence; - int ret; + xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true); + ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt); + xe_tlb_inval_fence_wait(fence_ptr); - xe_tlb_inval_fence_init(tlb_inval, &fence, true); - ret = __xe_tlb_inval_ggtt(gt, &fence); - if (ret) - return ret; - - xe_tlb_inval_fence_wait(&fence); - } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { - struct xe_mmio *mmio = >->mmio; - - if (IS_SRIOV_VF(xe)) - return 0; - - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, - PVC_GUC_TLB_INV_DESC1_INVALIDATE); - xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0, - PVC_GUC_TLB_INV_DESC0_VALID); - } else { - xe_mmio_write32(mmio, GUC_TLB_INV_CR, - GUC_TLB_INV_CR_INVALIDATE); - } - xe_force_wake_put(gt_to_fw(gt), fw_ref); - } - - return 0; + return ret; } /** - * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range + * xe_tlb_inval_range() - Issue a TLB invalidation for an address range * @tlb_inval: TLB invalidation client * @fence: invalidation fence which will be signal on TLB invalidation * completion @@ -475,31 +311,12 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, u64 start, u64 end, u32 asid) { - struct xe_gt *gt = tlb_inval->private; - struct xe_device *xe = gt_to_xe(gt); - int ret; - - xe_gt_assert(gt, fence); - - /* Execlists not supported */ - if (xe->info.force_execlist) { - __inval_fence_signal(xe, fence); - return 0; - } - - mutex_lock(>->tlb_inval.seqno_lock); - xe_tlb_inval_fence_prep(fence); - - ret = send_tlb_inval_ppgtt(gt, start, end, asid, fence->seqno); - if (ret < 0) - inval_fence_signal_unlocked(xe, fence); - mutex_unlock(>->tlb_inval.seqno_lock); - - return ret; + return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt, + start, end, asid); } /** - * xe_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM + * xe_tlb_inval_vm() - Issue a TLB invalidation for a VM * @tlb_inval: TLB invalidation client * @vm: VM to invalidate * @@ -509,27 +326,22 @@ void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm) { struct xe_tlb_inval_fence fence; u64 range = 1ull << vm->xe->info.va_bits; - int ret; xe_tlb_inval_fence_init(tlb_inval, &fence, true); - - ret = xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid); - if (ret < 0) - return; - + xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid); xe_tlb_inval_fence_wait(&fence); } /** - * xe_tlb_inval_done_handler - TLB invalidation done handler - * @gt: gt + * xe_tlb_inval_done_handler() - TLB invalidation done handler + * @tlb_inval: TLB invalidation client * @seqno: seqno of invalidation that is done * * Update recv seqno, signal any TLB invalidation fences, and restart TDR */ -static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno) +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno) { - struct xe_device *xe = gt_to_xe(gt); + struct xe_device *xe = tlb_inval->xe; struct xe_tlb_inval_fence *fence, *next; unsigned long flags; @@ -548,77 +360,53 @@ static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno) * officially process the CT message like if racing against * process_g2h_msg(). */ - spin_lock_irqsave(>->tlb_inval.pending_lock, flags); - if (tlb_inval_seqno_past(gt, seqno)) { - spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); + spin_lock_irqsave(&tlb_inval->pending_lock, flags); + if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) { + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); return; } - WRITE_ONCE(gt->tlb_inval.seqno_recv, seqno); + WRITE_ONCE(tlb_inval->seqno_recv, seqno); list_for_each_entry_safe(fence, next, - >->tlb_inval.pending_fences, link) { + &tlb_inval->pending_fences, link) { trace_xe_tlb_inval_fence_recv(xe, fence); - if (!tlb_inval_seqno_past(gt, fence->seqno)) + if (!xe_tlb_inval_seqno_past(tlb_inval, fence->seqno)) break; - inval_fence_signal(xe, fence); + xe_tlb_inval_fence_signal(fence); } - if (!list_empty(>->tlb_inval.pending_fences)) + if (!list_empty(&tlb_inval->pending_fences)) mod_delayed_work(system_wq, - >->tlb_inval.fence_tdr, - tlb_timeout_jiffies(gt)); + &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); else - cancel_delayed_work(>->tlb_inval.fence_tdr); + cancel_delayed_work(&tlb_inval->fence_tdr); - spin_unlock_irqrestore(>->tlb_inval.pending_lock, flags); -} - -/** - * xe_guc_tlb_inval_done_handler - TLB invalidation done handler - * @guc: guc - * @msg: message indicating TLB invalidation done - * @len: length of message - * - * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any - * invalidation fences for seqno. Algorithm for this depends on seqno being - * received in-order and asserts this assumption. - * - * Return: 0 on success, -EPROTO for malformed messages. - */ -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len) -{ - struct xe_gt *gt = guc_to_gt(guc); - - if (unlikely(len != 1)) - return -EPROTO; - - xe_tlb_inval_done_handler(gt, msg[0]); - - return 0; + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); } static const char * -inval_fence_get_driver_name(struct dma_fence *dma_fence) +xe_inval_fence_get_driver_name(struct dma_fence *dma_fence) { return "xe"; } static const char * -inval_fence_get_timeline_name(struct dma_fence *dma_fence) +xe_inval_fence_get_timeline_name(struct dma_fence *dma_fence) { - return "inval_fence"; + return "tlb_inval_fence"; } static const struct dma_fence_ops inval_fence_ops = { - .get_driver_name = inval_fence_get_driver_name, - .get_timeline_name = inval_fence_get_timeline_name, + .get_driver_name = xe_inval_fence_get_driver_name, + .get_timeline_name = xe_inval_fence_get_timeline_name, }; /** - * xe_tlb_inval_fence_init - Initialize TLB invalidation fence + * xe_tlb_inval_fence_init() - Initialize TLB invalidation fence * @tlb_inval: TLB invalidation client * @fence: TLB invalidation fence to initialize * @stack: fence is stack variable @@ -631,15 +419,12 @@ void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, bool stack) { - struct xe_gt *gt = tlb_inval->private; + xe_pm_runtime_get_noresume(tlb_inval->xe); - xe_pm_runtime_get_noresume(gt_to_xe(gt)); - - spin_lock_irq(>->tlb_inval.lock); - dma_fence_init(&fence->base, &inval_fence_ops, - >->tlb_inval.lock, + spin_lock_irq(&tlb_inval->lock); + dma_fence_init(&fence->base, &inval_fence_ops, &tlb_inval->lock, dma_fence_context_alloc(1), 1); - spin_unlock_irq(>->tlb_inval.lock); + spin_unlock_irq(&tlb_inval->lock); INIT_LIST_HEAD(&fence->link); if (stack) set_bit(FENCE_STACK_BIT, &fence->base.flags); diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h index ab6f769c50be..554634dfd4e2 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval.h @@ -15,27 +15,32 @@ struct xe_guc; struct xe_vm; int xe_gt_tlb_inval_init_early(struct xe_gt *gt); -void xe_gt_tlb_inval_fini(struct xe_gt *gt); void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval); -int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); -void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm); int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence); +int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); +void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm); int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, u64 start, u64 end, u32 asid); -int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len); void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, bool stack); -void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence); +/** + * xe_tlb_inval_fence_wait() - TLB invalidiation fence wait + * @fence: TLB invalidation fence to wait on + * + * Wait on a TLB invalidiation fence until it signals, non interruptable + */ static inline void xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence) { dma_fence_wait(&fence->base, false); } +void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno); + #endif /* _XE_TLB_INVAL_ */ diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h index 6d14b9f17b91..8f8b060e9005 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -9,10 +9,75 @@ #include #include -/** struct xe_tlb_inval - TLB invalidation client */ +struct xe_tlb_inval; + +/** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */ +struct xe_tlb_inval_ops { + /** + * @all: Invalidate all TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*all)(struct xe_tlb_inval *tlb_inval, u32 seqno); + + /** + * @ggtt: Invalidate global translation TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*ggtt)(struct xe_tlb_inval *tlb_inval, u32 seqno); + + /** + * @ppgtt: Invalidate per-process translation TLBs + * @tlb_inval: TLB invalidation client + * @seqno: Seqno of TLB invalidation + * @start: Start address + * @end: End address + * @asid: Address space ID + * + * Return 0 on success, -ECANCELED if backend is mid-reset, error on + * failure + */ + int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start, + u64 end, u32 asid); + + /** + * @initialized: Backend is initialized + * @tlb_inval: TLB invalidation client + * + * Return: True if back is initialized, False otherwise + */ + bool (*initialized)(struct xe_tlb_inval *tlb_inval); + + /** + * @flush: Flush pending TLB invalidations + * @tlb_inval: TLB invalidation client + */ + void (*flush)(struct xe_tlb_inval *tlb_inval); + + /** + * @timeout_delay: Timeout delay for TLB invalidation + * @tlb_inval: TLB invalidation client + * + * Return: Timeout delay for TLB invalidation in jiffies + */ + long (*timeout_delay)(struct xe_tlb_inval *tlb_inval); +}; + +/** struct xe_tlb_inval - TLB invalidation client (frontend) */ struct xe_tlb_inval { /** @private: Backend private pointer */ void *private; + /** @xe: Pointer to Xe device */ + struct xe_device *xe; + /** @ops: TLB invalidation ops */ + const struct xe_tlb_inval_ops *ops; /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ #define TLB_INVALIDATION_SEQNO_MAX 0x100000 int seqno; From 81a45cb7ea31a59777af294de590eea66ab80691 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 8 Aug 2025 12:04:53 +0100 Subject: [PATCH 165/169] drm/xe/migrate: make MI_TLB_INVALIDATE conditional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When clearing VRAM we should be able to skip invalidating the TLBs if we are only using the identity map to access VRAM (which is the common case), since no modifications are made to PTEs on the fly. Also since we use huge 1G entries within the identity map, there should be a pretty decent chance that the next packet(s) (if also clears) can avoid a tree walk if we don't shoot down the TLBs, like if we have to process a long stream of clears. For normal moves/copies, we usually always end up with the src or dst being system memory, meaning we can't only rely on the identity map and will also need to emit PTEs and so will always require a TLB flush. v2: - Update commit to explain the situation for normal copies (Matt B) - Rebase on latest changes Signed-off-by: Matthew Auld Cc: Himal Prasad Ghimiray Cc: Thomas Hellström Cc: Matthew Brost Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20250808110452.467513-2-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 18 +++++++++++------- drivers/gpu/drm/xe/xe_ring_ops.c | 10 +++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 57e6d5a8ac39..9643442ef101 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -904,7 +904,7 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, goto err; } - xe_sched_job_add_migrate_flush(job, flush_flags); + xe_sched_job_add_migrate_flush(job, flush_flags | MI_INVALIDATE_TLB); if (!fence) { err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, DMA_RESV_USAGE_BOOKKEEP); @@ -1288,11 +1288,13 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m, size -= clear_L0; /* Preemption is enabled again by the ring ops. */ - if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) + if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) { xe_res_next(&src_it, clear_L0); - else - emit_pte(m, bb, clear_L0_pt, clear_vram, clear_only_system_ccs, - &src_it, clear_L0, dst); + } else { + emit_pte(m, bb, clear_L0_pt, clear_vram, + clear_only_system_ccs, &src_it, clear_L0, dst); + flush_flags |= MI_INVALIDATE_TLB; + } bb->cs[bb->len++] = MI_BATCH_BUFFER_END; update_idx = bb->len; @@ -1303,7 +1305,7 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m, if (xe_migrate_needs_ccs_emit(xe)) { emit_copy_ccs(gt, bb, clear_L0_ofs, true, m->cleared_mem_ofs, false, clear_L0); - flush_flags = MI_FLUSH_DW_CCS; + flush_flags |= MI_FLUSH_DW_CCS; } job = xe_bb_create_migration_job(m->q, bb, @@ -1638,6 +1640,8 @@ next_cmd: goto err_sa; } + xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); + if (ops->pre_commit) { pt_update->job = job; err = ops->pre_commit(pt_update); @@ -1863,7 +1867,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, goto err; } - xe_sched_job_add_migrate_flush(job, 0); + xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); mutex_lock(&m->job_mutex); xe_sched_job_arm(job); diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 5f15360d14bf..d71837773d6c 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -110,10 +110,10 @@ static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) return i; } -static int emit_flush_invalidate(u32 addr, u32 val, u32 *dw, int i) +static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) { - dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | - MI_FLUSH_IMM_DW; + dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | + MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; dw[i++] = addr | MI_FLUSH_DW_USE_GTT; dw[i++] = 0; @@ -410,13 +410,13 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); dw[i++] = preparser_disable(true); - i = emit_flush_invalidate(saddr, seqno, dw, i); + i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i); dw[i++] = preparser_disable(false); i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, - MI_INVALIDATE_TLB | job->migrate_flush_flags, + job->migrate_flush_flags, dw, i); i = emit_user_interrupt(dw, i); From 12267f831fa6fe679019d09c48089941908b7cf3 Mon Sep 17 00:00:00 2001 From: Marcin Bernatowicz Date: Thu, 14 Aug 2025 16:59:50 +0200 Subject: [PATCH 166/169] drm/xe/tests: Make cross-device dma-buf BOs CPU-visible on small BAR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small-BAR systems (e.g., SR-IOV VFs in VMs) expose only a subset of VRAM via PCI/BAR. Exporting a BO outside that window fails, and the selftests also do CPU fill/verify. Set XE_BO_FLAG_NEEDS_CPU_ACCESS for cross-device variants to force CPU-mappable placement and keep tests reliable. Large-BAR/P2P setups are unaffected. Suggested-by: Matthew Auld Signed-off-by: Marcin Bernatowicz Cc: Matthew Auld Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20250814145950.430231-1-marcin.bernatowicz@linux.intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/tests/xe_dma_buf.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index cde9530bef8c..3c5ad8cc65a0 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -204,7 +204,7 @@ static const struct dma_buf_attach_ops nop2p_attach_ops = { static const struct dma_buf_test_params test_params[] = { {.mem_mask = XE_BO_FLAG_VRAM0, .attach_ops = &xe_dma_buf_attach_ops}, - {.mem_mask = XE_BO_FLAG_VRAM0, + {.mem_mask = XE_BO_FLAG_VRAM0 | XE_BO_FLAG_NEEDS_CPU_ACCESS, .attach_ops = &xe_dma_buf_attach_ops, .force_different_devices = true}, @@ -236,7 +236,8 @@ static const struct dma_buf_test_params test_params[] = { {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, .attach_ops = &xe_dma_buf_attach_ops}, - {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0, + {.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0 | + XE_BO_FLAG_NEEDS_CPU_ACCESS, .attach_ops = &xe_dma_buf_attach_ops, .force_different_devices = true}, From ed906b7076d072fc0f1b358c868ff69cab69607c Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Mon, 25 Aug 2025 16:05:37 +0530 Subject: [PATCH 167/169] drm/xe/nvm: Use root tile mmio To allow initialization of nvm during early probe for future usecases, use root tile instead of root gt to access mmios, as gt is not yet initialized at early probe. v2: fix commit message (Lucas) Signed-off-by: Riana Tauro Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250825103537.2551837-1-riana.tauro@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_nvm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c index 61b0a1531a53..9ca4a5ae1693 100644 --- a/drivers/gpu/drm/xe/xe_nvm.c +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -39,17 +39,17 @@ static void xe_nvm_release_dev(struct device *dev) static bool xe_nvm_non_posted_erase(struct xe_device *xe) { - struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_mmio *mmio = xe_root_tile_mmio(xe); if (xe->info.platform != XE_BATTLEMAGE) return false; - return !(xe_mmio_read32(>->mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) & + return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) & NVM_NON_POSTED_ERASE_CHICKEN_BIT); } static bool xe_nvm_writable_override(struct xe_device *xe) { - struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_mmio *mmio = xe_root_tile_mmio(xe); bool writable_override; resource_size_t base; @@ -72,7 +72,7 @@ static bool xe_nvm_writable_override(struct xe_device *xe) } writable_override = - !(xe_mmio_read32(>->mmio, HECI_FWSTS2(base)) & + !(xe_mmio_read32(mmio, HECI_FWSTS2(base)) & HECI_FW_STATUS_2_NVM_ACCESS_MODE); if (writable_override) drm_info(&xe->drm, "NVM access overridden by jumper\n"); From eeb8117f5f1c2e8e625e5cb39dbccd21d395caad Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 28 Aug 2025 12:45:16 +0530 Subject: [PATCH 168/169] drm/xe/uapi: Fix kernel-doc formatting for madvise and vma_query Correct kernel-doc formatting issues in the UAPI definitions for madvise and VMA query interfaces to resolve docutils warnings during documentation build. Fixes: 418807860e94 ("drm/xe/uapi: Add UAPI for querying VMA count and memory attributes") Fixes: 231bb0ee7aa5 ("drm/xe/uapi: Add madvise interface") Cc: Matthew Brost Cc: Lucas De Marchi Signed-off-by: Himal Prasad Ghimiray Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20250828071516.3838110-1-himal.prasad.ghimiray@intel.com Signed-off-by: Lucas De Marchi --- include/uapi/drm/xe_drm.h | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 7dedd45ab995..40ff19f52a8d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1997,23 +1997,23 @@ struct drm_xe_query_eu_stall { * union member is used to provide additional parameters for @type. * * Supported attribute types: - * - DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: Set preferred memory location. - * - DRM_XE_MEM_RANGE_ATTR_ATOMIC: Set atomic access policy. - * - DRM_XE_MEM_RANGE_ATTR_PAT: Set page attribute table index. + * - DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: Set preferred memory location. + * - DRM_XE_MEM_RANGE_ATTR_ATOMIC: Set atomic access policy. + * - DRM_XE_MEM_RANGE_ATTR_PAT: Set page attribute table index. * * Example: * * .. code-block:: C * - * struct drm_xe_madvise madvise = { - *          .vm_id = vm_id, - *          .start = 0x100000, - *          .range = 0x2000, - *          .type = DRM_XE_MEM_RANGE_ATTR_ATOMIC, - *         .atomic_val = DRM_XE_ATOMIC_DEVICE, - * }; + * struct drm_xe_madvise madvise = { + * .vm_id = vm_id, + * .start = 0x100000, + * .range = 0x2000, + * .type = DRM_XE_MEM_RANGE_ATTR_ATOMIC, + * .atomic_val = DRM_XE_ATOMIC_DEVICE, + * }; * - * ioctl(fd, DRM_IOCTL_XE_MADVISE, &madvise); + * ioctl(fd, DRM_IOCTL_XE_MADVISE, &madvise); * */ struct drm_xe_madvise { @@ -2042,12 +2042,12 @@ struct drm_xe_madvise { * Used when @type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC * * Supported values for @preferred_mem_loc.devmem_fd: - * - DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE: set vram of faulting tile as preferred loc - * - DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM: set smem as preferred loc + * - DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE: set vram of fault tile as preferred loc + * - DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM: set smem as preferred loc * * Supported values for @preferred_mem_loc.migration_policy: - * - DRM_XE_MIGRATE_ALL_PAGES - * - DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES + * - DRM_XE_MIGRATE_ALL_PAGES + * - DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES */ struct { #define DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE 0 @@ -2073,12 +2073,12 @@ struct drm_xe_madvise { * Used when @type == DRM_XE_MEM_RANGE_ATTR_ATOMIC. * * Supported values for @atomic.val: - * - DRM_XE_ATOMIC_UNDEFINED: Undefined or default behaviour - * Support both GPU and CPU atomic operations for system allocator - * Support GPU atomic operations for normal(bo) allocator - * - DRM_XE_ATOMIC_DEVICE: Support GPU atomic operations - * - DRM_XE_ATOMIC_GLOBAL: Support both GPU and CPU atomic operations - * - DRM_XE_ATOMIC_CPU: Support CPU atomic + * - DRM_XE_ATOMIC_UNDEFINED: Undefined or default behaviour. + * Support both GPU and CPU atomic operations for system allocator. + * Support GPU atomic operations for normal(bo) allocator. + * - DRM_XE_ATOMIC_DEVICE: Support GPU atomic operations. + * - DRM_XE_ATOMIC_GLOBAL: Support both GPU and CPU atomic operations. + * - DRM_XE_ATOMIC_CPU: Support CPU atomic only, no GPU atomics supported. */ struct { #define DRM_XE_ATOMIC_UNDEFINED 0 @@ -2196,6 +2196,7 @@ struct drm_xe_mem_range_attr { * Example: * * .. code-block:: C + * * struct drm_xe_vm_query_mem_range_attr query = { * .vm_id = vm_id, * .start = 0x100000, From 1047bd82794a1eab64d643f196d09171ce983f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 28 Aug 2025 15:48:37 +0200 Subject: [PATCH 169/169] drm/xe: Fix incorrect migration of backed-up object to VRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If an object is backed up to shmem it is incorrectly identified as not having valid data by the move code. This means moving to VRAM skips the -EMULTIHOP step and the bo is cleared. This causes all sorts of weird behaviour on DGFX if an already evicted object is targeted by the shrinker. Fix this by using ttm_tt_is_swapped() to identify backed-up objects. Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/5996 Fixes: 00c8efc3180f ("drm/xe: Add a shrinker for xe bos") Cc: Matthew Brost Cc: Matthew Auld Cc: # v6.15+ Signed-off-by: Thomas Hellström Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20250828134837.5709-1-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_bo.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 7d1ff642b02a..4faf15d5fa6d 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -823,8 +823,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, return ret; } - tt_has_data = ttm && (ttm_tt_is_populated(ttm) || - (ttm->page_flags & TTM_TT_FLAG_SWAPPED)); + tt_has_data = ttm && (ttm_tt_is_populated(ttm) || ttm_tt_is_swapped(ttm)); move_lacks_source = !old_mem || (handle_system_ccs ? (!bo->ccs_cleared) : (!mem_type_is_vram(old_mem_type) && !tt_has_data));