From:   Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
To:     linux-kernel@vger.kernel.org, linux-pci@vger.kernel.org,
        linux-arm-kernel@lists.infradead.org, iommu@lists.linux.dev,
        joro@8bytes.org, bhelgaas@google.com, robin.murphy@arm.com,
        will@kernel.org
Cc:     jean-philippe@linaro.org, darren@os.amperecomputing.com,
        scott@os.amperecomputing.com, gankulkarni@os.amperecomputing.com
Subject: [PATCH 2/2] iommu/arm-smmu-v3: Configure STU of a PF if ATS is not enabled.
Date:   Mon, 27 Feb 2023 05:21:51 -0800
Message-Id: <20230227132151.1907480-3-gankulkarni@os.amperecomputing.com>
In-Reply-To: <20230227132151.1907480-1-gankulkarni@os.amperecomputing.com>
References: <20230227132151.1907480-1-gankulkarni@os.amperecomputing.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?ROLTypb+l8NKAvLLGiOvozbTfjxtCEEFsZCEhE1iewhz5QTaGDfuX/pvLxYD?=
 =?us-ascii?Q?y5QNhMm0X7BGS7vaw7LMa3fSJlbix8fViJWZhA+G7qYsWfNUmYSM3KIEpu5Q?=
 =?us-ascii?Q?vpX35qybCvLXBMQonVRAESoGthtR/NkbD3wKaWYzzRqyckzWxGpG9RKsmuzV?=
 =?us-ascii?Q?Hit2Sha+Chbj5gviSqqpz+RmVGsfhFkI3PV6TEoyzRULMYaQMX7xUk56kXFU?=
 =?us-ascii?Q?9TElezVQfHnH/g0O1jf+KPPi4r12r3dBdUJd6IvAxOpwXKyF1bpIxkrSs6QE?=
 =?us-ascii?Q?MQuHFecx0YDjaksn5jR0o+pulK4yiatQ1UDPcMaOmirLdSakHbh+WNDPjnoU?=
 =?us-ascii?Q?S2+6K725YsgxaCoXIO65RvbrFjEH8HC/vFzDOhKVGhg/MCxB4+5EgueRz8rE?=
 =?us-ascii?Q?mlcB6j6S+GVVzCTCV4abtyiBPWDPmFYnxMGwyzERkh+v46erw9txlXejSpIU?=
 =?us-ascii?Q?sze1tvbd6LJhN/OCYJ4Geo74QyxbxybS4GXsinyfuhvOUAO+KAIvle/fZ+7v?=
 =?us-ascii?Q?FmtiEidGLIsmFY/mTvGRGM8dXqkBhKqozYxwZJ6Phk90NN6z4dU1OHFNGcFQ?=
 =?us-ascii?Q?1XzdDuj1biPkQJSkNwWmep0ulSX0pVtn0tS1j/OVwsGgTtnfpxtb/tQ3HLzX?=
 =?us-ascii?Q?+JbOG2a7lArBTk69ziHnNHnzJdfIXKOAbnvspUaw2AjgqYhs3PVvxAIc36QV?=
 =?us-ascii?Q?TKhM5kdHSj/7er3QgFazfoJZ3PhnNN/IATCudKZlMjpneqHBdv4ahjhte90I?=
 =?us-ascii?Q?m++t1tOI1DPQJ1Q9pYY4gfmmzI32f4G7xnkWlN/QXNNuXjiqYx8WiRabblAu?=
 =?us-ascii?Q?q7AIYGVZwu92uBfLV9CzA6FtPgGpPdOU70Wb75BKYfsU+eLs1cXpnokl2AJo?=
 =?us-ascii?Q?9PbOSOGun9+qd68SPd3vmMrM3uMhdkYjTQvPth/R+YoRtseyvBp7k+x7Buaf?=
 =?us-ascii?Q?uDMYfYGgMJFvfq659Dpdd4iH155CnzvPkqmOnA/2UnmbmgE5vjUSQeH8I5Kf?=
 =?us-ascii?Q?+kY32LWu0cTh79xqdyXU9khqlILaAR6AU1q7qCJKd9lpBLeoxjsDxTMN0yrH?=
 =?us-ascii?Q?xv810MvAYyzi2D4nhNjX8dxgDhpZOO1Eufm6nEm/2H+nsEb5fMjivPRs1Dfc?=
 =?us-ascii?Q?amAWWnnwuzpB10QZhbZWh6DcRvitD5kEQLEHxMWnak+2e05+M14+ZQjIImqz?=
 =?us-ascii?Q?OvWju4TunA7p8SvGUMAL+xjJK3mfhTiSil5Kfz2htQXbExpm7oOLBU7LNjVh?=
 =?us-ascii?Q?VWYLTlK/LQtp14Vv11bfvGd4yvGiaXYEi++m9+KTJln9bfBHU/3Ut6VMb0XS?=
 =?us-ascii?Q?Gt356GIZ0iISDPneZOWCxfvFeazg2ukEA6AN596dUib9fZz/S8ABvCs5F6EE?=
 =?us-ascii?Q?KL9IlCQ6UeTBfDbKSrQrKStLYhZ2z3TiTZq1fbg/veJm4x5UupbpHQUTY0Mo?=
 =?us-ascii?Q?EZ4Ptm7k2/jNxjG+jn2fDL5Dr7MZ7v3moTXnU5qUY7hZ8b6nhy2PnzzVj5f+?=
 =?us-ascii?Q?Afo3chC/QpXQ8GF0MRq3sEsdKayvVhsfY2gyI+XoA//gcQ5/p0VNhovHUv/x?=
 =?us-ascii?Q?T/E93skZ9xcte5zqD96f4rSHb4ZwqpIEaxONsXLKThKYOxnC9Slft5D+ZKhV?=
 =?us-ascii?Q?shpQjAKuYmQsoFHRghZk6i4=3D?=
X-OriginatorOrg: os.amperecomputing.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 0ed7eea0-01be-4e14-a65f-08db18c5a23c
X-MS-Exchange-CrossTenant-AuthSource: DM8PR01MB6824.prod.exchangelabs.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 27 Feb 2023 13:22:11.3658
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 3bc2b170-fd94-476d-b0ce-4229bdc904a7
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: B6mlNueOaw4Z6XRfsR/bpERinI/5xJ7LNgu9lyfvMretx1cTyyaFX87zFnJU1KATKpIcAYTnQcaQLsF0dRPJsf23i6Ra4IRM7o5sTHQZ/48i3vNNA0cCdyaXOCdJnCuu
X-MS-Exchange-Transport-CrossTenantHeadersStamped: MWHPR01MB2719
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

When the host kernel is booted with iommu passthrough mode, PF and VFs
are enumerated with iommu/smmu domain set to bypass mode.
In bypass mode, ATS is not enabled on all VFs and associated PF.
When VFs are attached to a VM, the corresponding iommu domain is set to
SMMU translation mode and smmu-v3 driver try to enable the ATS and fails
due to invalid STU of a PF.

Adding a fix to configure STU of a PF while enumerating in passthrough
mode.

Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 25 ++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f2425b0f0cd6..b218ef0bf001 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2292,6 +2292,23 @@ static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
 	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
 }
 
+static void arm_smmu_ats_stu_init(struct arm_smmu_master *master)
+{
+	size_t stu;
+	struct pci_dev *pdev;
+	struct arm_smmu_device *smmu = master->smmu;
+
+	if (master->ats_enabled)
+		return;
+
+	/* Smallest Translation Unit: log2 of the smallest supported granule */
+	stu = __ffs(smmu->pgsize_bitmap);
+	pdev = to_pci_dev(master->dev);
+
+	if (pci_ats_stu_configure(pdev, stu))
+		dev_err(master->dev, "Failed to configure ATS STU (%zu)\n", stu);
+}
+
 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 {
 	size_t stu;
@@ -2404,6 +2421,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 	struct arm_smmu_master *master;
+	bool ats_supported;
 
 	if (!fwspec)
 		return -ENOENT;
@@ -2446,9 +2464,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	}
 
 	master->domain = smmu_domain;
+	ats_supported = arm_smmu_ats_supported(master);
 
 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
-		master->ats_enabled = arm_smmu_ats_supported(master);
+		master->ats_enabled = ats_supported;
 
 	arm_smmu_install_ste_for_dev(master);
 
@@ -2458,6 +2477,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 
 	arm_smmu_enable_ats(master);
 
+	/* Configure ATS STU of a PF in passthrough */
+	if (!master->ats_enabled && ats_supported)
+		arm_smmu_ats_stu_init(master);
+
 out_unlock:
 	mutex_unlock(&smmu_domain->init_mutex);
 	return ret;
-- 
2.38.1