[24.04_linux-nvidia-6.17-next] ats always on v4

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

@@ -1472,8 +1472,11 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
 	if (!arm_smmu_cdtab_allocated(&master->cd_table))
 		return;
 	cdptr = arm_smmu_get_cd_ptr(master, ssid);
-	if (!cdptr)
+	if (!cdptr) {
+		/* Only ats_always_on allows a NULL CD on default substream */
+		WARN_ON(!master->ats_always_on || ssid);
 		return;
+	}
 	arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
 }
 
@@ -3270,14 +3273,16 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
 static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
 				    struct device *dev,
 				    struct arm_smmu_ste *ste,
-				    unsigned int s1dss, bool ats_always_on)
+				    unsigned int s1dss)
 {
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 	struct arm_smmu_attach_state state = {
 		.master = master,
 		.old_domain = iommu_get_domain_for_dev(dev),
 		.ssid = IOMMU_NO_PASID,
 	};
+	bool ats_always_on = master->ats_always_on &&
+			     s1dss != STRTAB_STE_1_S1DSS_TERMINATE;
 
 	/*
 	 * Do not allow any ASID to be changed while are working on the STE,
@@ -3323,8 +3328,7 @@ static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
 
 	arm_smmu_master_clear_vmaster(master);
 	arm_smmu_make_bypass_ste(master->smmu, &ste);
-	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS,
-				master->ats_always_on);
+	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
 	return 0;
 }
 
@@ -3345,8 +3349,7 @@ static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
 
 	arm_smmu_master_clear_vmaster(master);
 	arm_smmu_make_abort_ste(&ste);
-	arm_smmu_attach_dev_ste(domain, dev, &ste,
-				STRTAB_STE_1_S1DSS_TERMINATE, false);
+	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_TERMINATE);
 	return 0;
 }
 
@@ -3588,12 +3591,14 @@ static int arm_smmu_master_prepare_ats(struct arm_smmu_master *master)
 {
 	bool s1p = master->smmu->features & ARM_SMMU_FEAT_TRANS_S1;
 	unsigned int stu = __ffs(master->smmu->pgsize_bitmap);
-	struct pci_dev *pdev = to_pci_dev(master->dev);
+	struct pci_dev *pdev;
 	int ret;
 
 	if (!arm_smmu_ats_supported(master))
 		return 0;
 
+	pdev = to_pci_dev(master->dev);
+
 	if (!pci_ats_always_on(pdev))
 		goto out_prepare;
 
@@ -3674,6 +3679,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 
 err_disable_pasid:
 	arm_smmu_disable_pasid(master);
+	arm_smmu_remove_master(master);
 err_free_master:
 	kfree(master);
 	return ERR_PTR(ret);

drivers/pci/ats.c

@@ -210,7 +210,8 @@ int pci_ats_page_aligned(struct pci_dev *pdev)
  * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host
  * by means of an ATS request on CXL.io.
  *
- * In other world, CXL.cache devices cannot access physical memory without ATS.
+ * In other words, CXL.cache devices cannot access host physical memory without
+ * ATS.
  */
 static bool pci_cxl_ats_always_on(struct pci_dev *pdev)
 {
@@ -222,19 +223,17 @@ static bool pci_cxl_ats_always_on(struct pci_dev *pdev)
 	if (!offset)
 		return false;
 
-	pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap);
-	if (cap & PCI_DVSEC_CXL_CACHE_CAPABLE)
-		return true;
+	if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap))
+		return false;
 
-	return false;
+	return cap & PCI_DVSEC_CXL_CACHE_CAPABLE;
 }
 
 /**
  * pci_ats_always_on - Whether the PCI device requires ATS to be always enabled
  * @pdev: the PCI device
  *
- * Returns true, if the PCI device requires non-PASID ATS function on an IOMMU
- * bypassed configuration.
+ * Returns true, if the PCI device requires ATS for basic functional operation.
  */
 bool pci_ats_always_on(struct pci_dev *pdev)
 {

drivers/pci/quirks.c

@@ -5689,23 +5689,38 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1459, quirk_intel_e2000_no_ats);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145a, quirk_intel_e2000_no_ats);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x145c, quirk_intel_e2000_no_ats);
 
+static bool quirk_nvidia_gpu_ats_always_on(struct pci_dev *pdev)
+{
+	switch (pdev->device) {
+	case 0x2e00 ... 0x2e3f: /* GB20B */
+		return true;
+	}
+	return false;
+}
+
 static const struct pci_dev_ats_always_on {
 	u16 vendor;
 	u16 device;
+	bool (*ats_always_on)(struct pci_dev *dev);
 } pci_dev_ats_always_on[] = {
-	{ PCI_VENDOR_ID_NVIDIA, 0x2e12, },
-	{ PCI_VENDOR_ID_NVIDIA, 0x2e2a, },
-	{ PCI_VENDOR_ID_NVIDIA, 0x2e2b, },
+	/* NVIDIA GPUs */
+	{ PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, quirk_nvidia_gpu_ats_always_on },
+	/* NVIDIA CX10 Family NVlink-C2C */
+	{ PCI_VENDOR_ID_MELLANOX, 0x2101, NULL },
 	{ 0 }
 };
 
-/* Some non-CXL devices support ATS on RID when it is IOMMU-bypassed */
+/* Some pre-CXL devices require ATS when it is IOMMU-bypassed */
 bool pci_dev_specific_ats_always_on(struct pci_dev *pdev)
 {
 	const struct pci_dev_ats_always_on *i;
 
 	for (i = pci_dev_ats_always_on; i->vendor; i++) {
-		if (i->vendor == pdev->vendor && i->device == pdev->device)
+		if (i->vendor != pdev->vendor)
+			continue;
+		if (i->ats_always_on && i->ats_always_on(pdev))
+			return true;
+		if (!i->ats_always_on && i->device == pdev->device)
 			return true;
 	}