From a80cf966a04bf699c459e6335656339b12af0dd4 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 30 Apr 2026 02:40:45 -0700 Subject: [PATCH 1/6] Revert "NVIDIA: VR: SAUCE: iommu/arm-smmu-v3: Allow ATS to be always on" BugLink: https://bugs.launchpad.net/bugs/2150727 This reverts commit 11d6b22546c50c28e16c79e3c24be23839a765c1. Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 75 +++------------------ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 - 2 files changed, 11 insertions(+), 65 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1923b113cddf8..58524ab2f775d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1472,7 +1472,7 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) if (!arm_smmu_cdtab_allocated(&master->cd_table)) return; cdptr = arm_smmu_get_cd_ptr(master, ssid); - if (!cdptr) + if (WARN_ON(!cdptr)) return; arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } @@ -1486,22 +1486,6 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; cd_table->s1cdmax = master->ssid_bits; - - /* - * When a device doesn't support PASID (non default SSID), ssid_bits is - * set to 0. This also sets S1CDMAX to 0, which disables the substreams - * and ignores the S1DSS field. - * - * On the other hand, if a device demands ATS to be always on even when - * its default substream is IOMMU bypassed, it has to use EATS that is - * only effective with an STE (CFG=S1translate, S1DSS=Bypass). For such - * use cases, S1CDMAX has to be !0, in order to make use of S1DSS/EATS. - * - * Set S1CDMAX no lower than 1. This would add a dummy substream in the - * CD table but it should never be used by an actual CD. - */ - if (master->ats_always_on) - cd_table->s1cdmax = max_t(u8, cd_table->s1cdmax, 1); max_contexts = 1 << cd_table->s1cdmax; if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || @@ -3255,8 +3239,7 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, * When the last user of the CD table goes away downgrade the STE back * to a non-cd_table one. */ - if (!master->ats_always_on && - !arm_smmu_ssids_in_use(&master->cd_table)) { + if (!arm_smmu_ssids_in_use(&master->cd_table)) { struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev); @@ -3270,7 +3253,7 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, struct device *dev, struct arm_smmu_ste *ste, - unsigned int s1dss, bool ats_always_on) + unsigned int s1dss) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct arm_smmu_attach_state state = { @@ -3289,7 +3272,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, * If the CD table is not in use we can use the provided STE, otherwise * we use a cdtable STE with the provided S1DSS. */ - if (ats_always_on || arm_smmu_ssids_in_use(&master->cd_table)) { + if (arm_smmu_ssids_in_use(&master->cd_table)) { /* * If a CD table has to be present then we need to run with ATS * on because we have to assume a PASID is using ATS. For @@ -3323,8 +3306,7 @@ static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, arm_smmu_master_clear_vmaster(master); arm_smmu_make_bypass_ste(master->smmu, &ste); - arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS, - master->ats_always_on); + arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS); return 0; } @@ -3345,8 +3327,7 @@ static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, arm_smmu_master_clear_vmaster(master); arm_smmu_make_abort_ste(&ste); - arm_smmu_attach_dev_ste(domain, dev, &ste, - STRTAB_STE_1_S1DSS_TERMINATE, false); + arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_TERMINATE); return 0; } @@ -3584,40 +3565,6 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->streams); } -static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master) -{ - bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1; - unsigned int stu = __ffs(master->smmu->pgsize_bitmap); - struct pci_dev *pdev = to_pci_dev(master->dev); - int ret; - - if (!arm_smmu_ats_supported(master)) - return 0; - - if (!pci_ats_always_on(pdev)) - goto out_prepare; - - /* - * S1DSS is required for ATS to be always on for identity domain cases. - * However, the S1DSS field is ignored if !IDR0_S1P or !IDR1_SSIDSIZE. - */ - if (!s1p || !master->smmu->ssid_bits) { - dev_info_once(master->dev, - "SMMU doesn't support ATS to be always on\n"); - goto out_prepare; - } - - master->ats_always_on = true; - - ret = arm_smmu_alloc_cd_tables(master); - if (ret) - return ret; - -out_prepare: - pci_prepare_ats(pdev, stu); - return 0; -} - static struct iommu_device *arm_smmu_probe_device(struct device *dev) { int ret; @@ -3666,14 +3613,14 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) smmu->features & ARM_SMMU_FEAT_STALL_FORCE) master->stall_enabled = true; - ret = arm_smmu_master_prepare_ats(master); - if (ret) - goto err_disable_pasid; + if (dev_is_pci(dev)) { + unsigned int stu = __ffs(smmu->pgsize_bitmap); + + pci_prepare_ats(to_pci_dev(dev), stu); + } return &smmu->iommu; -err_disable_pasid: - arm_smmu_disable_pasid(master); err_free_master: kfree(master); return ERR_PTR(ret); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 892d575ba7df9..f7fdca1efc473 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -861,7 +861,6 @@ struct arm_smmu_master { bool ats_enabled : 1; bool ste_ats_enabled : 1; bool stall_enabled; - bool ats_always_on; unsigned int ssid_bits; unsigned int iopf_refcount; u16 partid; From f684ff0b73198ad3ca47bfc2433cf17cf4abdcdf Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 30 Apr 2026 02:40:45 -0700 Subject: [PATCH 2/6] Revert "NVIDIA: VR: SAUCE: PCI: Allow ATS to be always on for non-CXL NVIDIA GPUs" BugLink: https://bugs.launchpad.net/bugs/2150727 This reverts commit 208b48c1e520001d08e8abe6ed49e0f7e2e5aa24. Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/pci/ats.c | 3 +-- drivers/pci/pci.h | 9 --------- drivers/pci/quirks.c | 23 ----------------------- 3 files changed, 1 insertion(+), 34 deletions(-) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index ae3152be018a7..93060fdc0d3c0 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -245,8 +245,7 @@ bool pci_ats_always_on(struct pci_dev *pdev) if (pdev->is_virtfn) pdev = pci_physfn(pdev); - return pci_cxl_ats_always_on(pdev) || - pci_dev_specific_ats_always_on(pdev); + return pci_cxl_ats_always_on(pdev); } EXPORT_SYMBOL_GPL(pci_ats_always_on); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 6167e0e204ade..11e9caaaeb010 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -957,15 +957,6 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe) } #endif -#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_PCI_ATS) -bool pci_dev_specific_ats_always_on(struct pci_dev *dev); -#else -static inline bool pci_dev_specific_ats_always_on(struct pci_dev *dev) -{ - return false; -} -#endif - #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64) int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment, struct resource *res); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 1703b98f5c406..65bfd7a7f7797 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5688,29 +5688,6 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1457, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, quirk_intel_e2000_no_ats); - -static const struct pci_dev_ats_always_on { - u16 vendor; - u16 device; -} pci_dev_ats_always_on[] = { - { PCI_VENDOR_ID_NVIDIA, 0x2e12, }, - { PCI_VENDOR_ID_NVIDIA, 0x2e2a, }, - { PCI_VENDOR_ID_NVIDIA, 0x2e2b, }, - { 0 } -}; - -/* Some non-CXL devices support ATS on RID when it is IOMMU-bypassed */ -bool pci_dev_specific_ats_always_on(struct pci_dev *pdev) -{ - const struct pci_dev_ats_always_on *i; - - for (i = pci_dev_ats_always_on; i->vendor; i++) { - if (i->vendor == pdev->vendor && i->device == pdev->device) - return true; - } - - return false; -} #endif /* CONFIG_PCI_ATS */ /* Freescale PCIe doesn't support MSI in RC mode */ From 9ce8c31158753fc799d91a4e1df37ed7bb4842b1 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 30 Apr 2026 02:40:45 -0700 Subject: [PATCH 3/6] Revert "NVIDIA: VR: SAUCE: PCI: Allow ATS to be always on for CXL.cache capable devices" BugLink: https://bugs.launchpad.net/bugs/2150727 This reverts commit 967f9fe2d7279bb9f61f993c2f788bcbcdfc69bf. Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/pci/ats.c | 44 ----------------------------------------- include/linux/pci-ats.h | 3 --- 2 files changed, 47 deletions(-) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 93060fdc0d3c0..ec6c8dbdc5e9c 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -205,50 +205,6 @@ int pci_ats_page_aligned(struct pci_dev *pdev) return 0; } -/* - * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on - * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host - * by means of an ATS request on CXL.io. - * - * In other world, CXL.cache devices cannot access physical memory without ATS. - */ -static bool pci_cxl_ats_always_on(struct pci_dev *pdev) -{ - int offset; - u16 cap; - - offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, - PCI_DVSEC_CXL_DEVICE); - if (!offset) - return false; - - pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap); - if (cap & PCI_DVSEC_CXL_CACHE_CAPABLE) - return true; - - return false; -} - -/** - * pci_ats_always_on - Whether the PCI device requires ATS to be always enabled - * @pdev: the PCI device - * - * Returns true, if the PCI device requires non-PASID ATS function on an IOMMU - * bypassed configuration. - */ -bool pci_ats_always_on(struct pci_dev *pdev) -{ - if (pci_ats_disabled() || !pci_ats_supported(pdev)) - return false; - - /* A VF inherits its PF's requirement for ATS function */ - if (pdev->is_virtfn) - pdev = pci_physfn(pdev); - - return pci_cxl_ats_always_on(pdev); -} -EXPORT_SYMBOL_GPL(pci_ats_always_on); - #ifdef CONFIG_PCI_PRI void pci_pri_init(struct pci_dev *pdev) { diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index d14ba727d38b3..75c6c86cf09dc 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -12,7 +12,6 @@ int pci_prepare_ats(struct pci_dev *dev, int ps); void pci_disable_ats(struct pci_dev *dev); int pci_ats_queue_depth(struct pci_dev *dev); int pci_ats_page_aligned(struct pci_dev *dev); -bool pci_ats_always_on(struct pci_dev *dev); #else /* CONFIG_PCI_ATS */ static inline bool pci_ats_supported(struct pci_dev *d) { return false; } @@ -25,8 +24,6 @@ static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } -static inline bool pci_ats_always_on(struct pci_dev *dev) -{ return false; } #endif /* CONFIG_PCI_ATS */ #ifdef CONFIG_PCI_PRI From ac5a5f2778407f46b2aacc32dc91fe7ce4406513 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sun, 26 Apr 2026 22:54:00 -0700 Subject: [PATCH 4/6] NVIDIA: VR: SAUCE: PCI: Allow ATS to be always on for CXL.cache capable devices BugLink: https://bugs.launchpad.net/bugs/2150727 Controlled by the IOMMU driver, ATS is usually enabled "on demand" when a given PASID on a device is attached to an I/O page table. This is working even when a device has no translation on its RID (i.e., the RID is IOMMU bypassed). However, certain PCIe devices require non-PASID ATS on their RID even when the RID is IOMMU bypassed. Call this "always on". For example, CXL spec r4.0 notes in sec 3.2.5.13 Memory Type on CXL.cache: "To source requests on CXL.cache, devices need to get the Host Physical Address (HPA) from the Host by means of an ATS request on CXL.io." In other words, the CXL.cache capability requires ATS; otherwise, it can't access host physical memory. Introduce a new pci_ats_always_on() helper for the IOMMU driver to scan a PCI device and shift ATS policies between "on demand" and "always on". Add the support for CXL.cache devices first. Pre-CXL devices will be added in quirks.c file. Note that pci_ats_always_on() validates against pci_ats_supported(), so we ensure that untrusted devices (e.g. external ports) will not be always on. This maintains the existing ATS security policy regarding potential side- channel attacks via ATS. Cc: linux-cxl@vger.kernel.org Suggested-by: Vikram Sethi Suggested-by: Jason Gunthorpe Reviewed-by: Jonathan Cameron Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Tested-by: Nirmoy Das Acked-by: Nirmoy Das Signed-off-by: Nicolin Chen Reviewed-by: Dave Jiang (backported from https://lore.kernel.org/r/f6734b9dad0050138676f11ecd14e9db1cf6b697.1777269009.git.nicolinc@nvidia.com) [Nirmoy: Adapt to already existing PCI_DVSEC_CXL_CACHE_CAPABLE.] Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/pci/ats.c | 43 +++++++++++++++++++++++++++++++++++++++++ include/linux/pci-ats.h | 3 +++ 2 files changed, 46 insertions(+) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index ec6c8dbdc5e9c..fc871858b65bc 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -205,6 +205,49 @@ int pci_ats_page_aligned(struct pci_dev *pdev) return 0; } +/* + * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on + * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host + * by means of an ATS request on CXL.io. + * + * In other words, CXL.cache devices cannot access host physical memory without + * ATS. + */ +static bool pci_cxl_ats_always_on(struct pci_dev *pdev) +{ + int offset; + u16 cap; + + offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, + PCI_DVSEC_CXL_DEVICE); + if (!offset) + return false; + + if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap)) + return false; + + return cap & PCI_DVSEC_CXL_CACHE_CAPABLE; +} + +/** + * pci_ats_always_on - Whether the PCI device requires ATS to be always enabled + * @pdev: the PCI device + * + * Returns true, if the PCI device requires ATS for basic functional operation. + */ +bool pci_ats_always_on(struct pci_dev *pdev) +{ + if (pci_ats_disabled() || !pci_ats_supported(pdev)) + return false; + + /* A VF inherits its PF's requirement for ATS function */ + if (pdev->is_virtfn) + pdev = pci_physfn(pdev); + + return pci_cxl_ats_always_on(pdev); +} +EXPORT_SYMBOL_GPL(pci_ats_always_on); + #ifdef CONFIG_PCI_PRI void pci_pri_init(struct pci_dev *pdev) { diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index 75c6c86cf09dc..d14ba727d38b3 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -12,6 +12,7 @@ int pci_prepare_ats(struct pci_dev *dev, int ps); void pci_disable_ats(struct pci_dev *dev); int pci_ats_queue_depth(struct pci_dev *dev); int pci_ats_page_aligned(struct pci_dev *dev); +bool pci_ats_always_on(struct pci_dev *dev); #else /* CONFIG_PCI_ATS */ static inline bool pci_ats_supported(struct pci_dev *d) { return false; } @@ -24,6 +25,8 @@ static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } +static inline bool pci_ats_always_on(struct pci_dev *dev) +{ return false; } #endif /* CONFIG_PCI_ATS */ #ifdef CONFIG_PCI_PRI From d3620020e6f74f3eed045725c627fd91c9ff7c2b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sun, 26 Apr 2026 22:54:01 -0700 Subject: [PATCH 5/6] NVIDIA: VR: SAUCE: PCI: Allow ATS to be always on for pre-CXL devices BugLink: https://bugs.launchpad.net/bugs/2150727 Some NVIDIA GPU/NIC devices, though they don't implement CXL config space, have many CXL-like properties. Call this kind "pre-CXL". Similar to CXL.cache capability, these pre-CXL devices also require the ATS function even when their RIDs are IOMMU bypassed, i.e. keep ATS "always on" v.s. "on demand" when a non-zero PASID line gets enabled in SVA use cases. Introduce pci_dev_specific_ats_always_on() quirk function to scan a list of IDs for these devices. Then, include it in pci_ats_always_on(). Suggested-by: Jason Gunthorpe Reviewed-by: Nirmoy Das Tested-by: Nirmoy Das Reviewed-by: Jonathan Cameron Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Nicolin Chen Reviewed-by: Dave Jiang (backported from https://lore.kernel.org/r/1a8cf5e88051ab5c10417edb94df598ecbc810cf.1777269009.git.nicolinc@nvidia.com) [Nirmoy: Apply after reverting older ATS always-on PCI quirk support.] Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/pci/ats.c | 3 ++- drivers/pci/pci.h | 9 +++++++++ drivers/pci/quirks.c | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index fc871858b65bc..3846447ea322f 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -244,7 +244,8 @@ bool pci_ats_always_on(struct pci_dev *pdev) if (pdev->is_virtfn) pdev = pci_physfn(pdev); - return pci_cxl_ats_always_on(pdev); + return pci_cxl_ats_always_on(pdev) || + pci_dev_specific_ats_always_on(pdev); } EXPORT_SYMBOL_GPL(pci_ats_always_on); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 11e9caaaeb010..6167e0e204ade 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -957,6 +957,15 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe) } #endif +#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_PCI_ATS) +bool pci_dev_specific_ats_always_on(struct pci_dev *dev); +#else +static inline bool pci_dev_specific_ats_always_on(struct pci_dev *dev) +{ + return false; +} +#endif + #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64) int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment, struct resource *res); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 65bfd7a7f7797..33d45d942ff7d 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5688,6 +5688,44 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1457, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, quirk_intel_e2000_no_ats); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, quirk_intel_e2000_no_ats); + +static bool quirk_nvidia_gpu_ats_always_on(struct pci_dev *pdev) +{ + switch (pdev->device) { + case 0x2e00 ... 0x2e3f: /* GB20B */ + return true; + } + return false; +} + +static const struct pci_dev_ats_always_on { + u16 vendor; + u16 device; + bool (*ats_always_on)(struct pci_dev *dev); +} pci_dev_ats_always_on[] = { + /* NVIDIA GPUs */ + { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, quirk_nvidia_gpu_ats_always_on }, + /* NVIDIA CX10 Family NVlink-C2C */ + { PCI_VENDOR_ID_MELLANOX, 0x2101, NULL }, + { 0 } +}; + +/* Some pre-CXL devices require ATS when it is IOMMU-bypassed */ +bool pci_dev_specific_ats_always_on(struct pci_dev *pdev) +{ + const struct pci_dev_ats_always_on *i; + + for (i = pci_dev_ats_always_on; i->vendor; i++) { + if (i->vendor != pdev->vendor) + continue; + if (i->ats_always_on && i->ats_always_on(pdev)) + return true; + if (!i->ats_always_on && i->device == pdev->device) + return true; + } + + return false; +} #endif /* CONFIG_PCI_ATS */ /* Freescale PCIe doesn't support MSI in RC mode */ From 8f9eaa3f3139f3234edb97fb5c87c9136304c2f9 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sun, 26 Apr 2026 22:54:02 -0700 Subject: [PATCH 6/6] NVIDIA: VR: SAUCE: iommu/arm-smmu-v3: Allow ATS to be always on BugLink: https://bugs.launchpad.net/bugs/2150727 When a device's default substream attaches to an identity domain, the SMMU driver currently sets the device's STE between two modes: Mode 1: Cfg=Translate, S1DSS=Bypass, EATS=1 Mode 2: Cfg=bypass (EATS is ignored by HW) When there is an active PASID (non-default substream), mode 1 is used. And when there is no PASID support or no active PASID, mode 2 is used. The driver will also downgrade an STE from mode 1 to mode 2, when the last active substream becomes inactive. However, there are PCIe devices that demand ATS to be always on. For these devices, their STEs have to use the mode 1 as HW ignores EATS with mode 2. Change the driver accordingly: - always use the mode 1 - never downgrade to mode 2 - allocate and retain a CD table (see note below) Note that these devices might not support PASID, i.e. doing non-PASID ATS. In such a case, the ssid_bits is set to 0. However, s1cdmax must be set to a !0 value in order to keep the S1DSS field effective. Thus, when a master requires ats_always_on, set its s1cdmax to at least 1, meaning that the CD table will have a dummy entry (SSID=1) that will never be used. Now for these devices, arm_smmu_cdtab_allocated() will always return true, v.s. false prior to this change. When its default substream is attached to an IDENTITY domain, its first CD is NULL in the table, which is a totally valid case. Thus, add "!master->ats_always_on" to the condition. Reviewed-by: Jonathan Cameron Tested-by: Nirmoy Das Acked-by: Nirmoy Das Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Nicolin Chen Reviewed-by: Dave Jiang (backported from https://lore.kernel.org/r/7403163ebf59380f88c7503b3adf0dae07428df8.1777269009.git.nicolinc@nvidia.com) [Nirmoy: Apply after reverting older ATS always-on arm-smmu-v3 support.] Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 75 ++++++++++++++++++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 58524ab2f775d..bd1757c784908 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1472,8 +1472,11 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) if (!arm_smmu_cdtab_allocated(&master->cd_table)) return; cdptr = arm_smmu_get_cd_ptr(master, ssid); - if (WARN_ON(!cdptr)) + if (!cdptr) { + /* Only ats_always_on allows a NULL CD on default substream */ + WARN_ON(!master->ats_always_on || ssid); return; + } arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } @@ -1486,6 +1489,22 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; cd_table->s1cdmax = master->ssid_bits; + + /* + * When a device doesn't support PASID (non default SSID), ssid_bits is + * set to 0. This also sets S1CDMAX to 0, which disables the substreams + * and ignores the S1DSS field. + * + * On the other hand, if a device demands ATS to be always on even when + * its default substream is IOMMU bypassed, it has to use EATS that is + * only effective with an STE (CFG=S1translate, S1DSS=Bypass). For such + * use cases, S1CDMAX has to be !0, in order to make use of S1DSS/EATS. + * + * Set S1CDMAX no lower than 1. This would add a dummy substream in the + * CD table but it should never be used by an actual CD. + */ + if (master->ats_always_on) + cd_table->s1cdmax = max_t(u8, cd_table->s1cdmax, 1); max_contexts = 1 << cd_table->s1cdmax; if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || @@ -3239,7 +3258,8 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, * When the last user of the CD table goes away downgrade the STE back * to a non-cd_table one. */ - if (!arm_smmu_ssids_in_use(&master->cd_table)) { + if (!master->ats_always_on && + !arm_smmu_ssids_in_use(&master->cd_table)) { struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev); @@ -3261,6 +3281,8 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, .old_domain = iommu_get_domain_for_dev(dev), .ssid = IOMMU_NO_PASID, }; + bool ats_always_on = master->ats_always_on && + s1dss != STRTAB_STE_1_S1DSS_TERMINATE; /* * Do not allow any ASID to be changed while are working on the STE, @@ -3272,7 +3294,7 @@ static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, * If the CD table is not in use we can use the provided STE, otherwise * we use a cdtable STE with the provided S1DSS. */ - if (arm_smmu_ssids_in_use(&master->cd_table)) { + if (ats_always_on || arm_smmu_ssids_in_use(&master->cd_table)) { /* * If a CD table has to be present then we need to run with ATS * on because we have to assume a PASID is using ATS. For @@ -3565,6 +3587,42 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->streams); } +static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master) +{ + bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1; + unsigned int stu = __ffs(master->smmu->pgsize_bitmap); + struct pci_dev *pdev; + int ret; + + if (!arm_smmu_ats_supported(master)) + return 0; + + pdev = to_pci_dev(master->dev); + + if (!pci_ats_always_on(pdev)) + goto out_prepare; + + /* + * S1DSS is required for ATS to be always on for identity domain cases. + * However, the S1DSS field is ignored if !IDR0_S1P or !IDR1_SSIDSIZE. + */ + if (!s1p || !master->smmu->ssid_bits) { + dev_info_once(master->dev, + "SMMU doesn't support ATS to be always on\n"); + goto out_prepare; + } + + master->ats_always_on = true; + + ret = arm_smmu_alloc_cd_tables(master); + if (ret) + return ret; + +out_prepare: + pci_prepare_ats(pdev, stu); + return 0; +} + static struct iommu_device *arm_smmu_probe_device(struct device *dev) { int ret; @@ -3613,14 +3671,15 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) smmu->features & ARM_SMMU_FEAT_STALL_FORCE) master->stall_enabled = true; - if (dev_is_pci(dev)) { - unsigned int stu = __ffs(smmu->pgsize_bitmap); - - pci_prepare_ats(to_pci_dev(dev), stu); - } + ret = arm_smmu_master_prepare_ats(master); + if (ret) + goto err_disable_pasid; return &smmu->iommu; +err_disable_pasid: + arm_smmu_disable_pasid(master); + arm_smmu_remove_master(master); err_free_master: kfree(master); return ERR_PTR(ret); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index f7fdca1efc473..892d575ba7df9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -861,6 +861,7 @@ struct arm_smmu_master { bool ats_enabled : 1; bool ste_ats_enabled : 1; bool stall_enabled; + bool ats_always_on; unsigned int ssid_bits; unsigned int iopf_refcount; u16 partid;