Received-SPF: pass (google.com: domain of linux-ext4-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
From:   Alex Sierra <alex.sierra@amd.com>
To:     akpm@linux-foundation.org, Felix.Kuehling@amd.com,
        linux-mm@kvack.org, rcampbell@nvidia.com,
        linux-ext4@vger.kernel.org, linux-xfs@vger.kernel.org
Cc:     amd-gfx@lists.freedesktop.org, dri-devel@lists.freedesktop.org,
        hch@lst.de, jgg@nvidia.com, jglisse@redhat.com
Subject: [PATCH v2 06/12] drm/amdkfd: add SPM support for SVM
Date:   Mon, 13 Sep 2021 11:15:58 -0500
Message-Id: <20210913161604.31981-7-alex.sierra@amd.com>
In-Reply-To: <20210913161604.31981-1-alex.sierra@amd.com>
References: <20210913161604.31981-1-alex.sierra@amd.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?PO2pryC76K3JPilDnjEuWaINdh0pgih37cgosoqupJGcjjYEuPSc581H3Rt5?=
 =?us-ascii?Q?PEw+qObgcDSuisLSZFOO5JvExgcwXP0huN0ZhhP5iqmG/m5te0eALHvnI5wc?=
 =?us-ascii?Q?yu4YvlhT45eKM5cptujnnHhXiySbMQFWNkEzUWziAjzBtFqsDIo6g1Tys+eP?=
 =?us-ascii?Q?49EG5XUYMlOk+sHziG+08DY+3MGn7f5gpU6yZ9oQPSqg7IyL7Kw7ny5Wda2g?=
 =?us-ascii?Q?4FzG2hSgdV8aX4z52GiL3E006KYpUd2bQYrhXmNRlC8T0FU7aOgYTjhQEeZP?=
 =?us-ascii?Q?4Mp7FI388ggd4ygMPtMgupatsMVD4uVY9zil8T9vdu4/4BJTixouwygIHnJQ?=
 =?us-ascii?Q?geglM6awT2DbtkPshfDHqLaF3mY1iG+S6mX6vL1mihh+V27yXSuZAbp3nGxo?=
 =?us-ascii?Q?Wo8g7/2e5uKadpdwS6CxPxeP6tGKf91smv9bJxBcW0rcxAK8ea0oxN0pvvN2?=
 =?us-ascii?Q?ObTl6DL7V5hs6Fh6yW3NoqiTFSS74L8K6VgNsRJ7SRWfJQMA7dcXOWzURYtm?=
 =?us-ascii?Q?2KU8Md+FyAwSJM1cr3E1ypePYfnE+lJALiWPgDEG2V8Dre2gA5avEy4WwkD8?=
 =?us-ascii?Q?wqlqSymoa+Jk6S8/rFi4zsGo9aDXlTdpyG8R0u4HAn+trKmqADl6Jo0HN4zB?=
 =?us-ascii?Q?6lHNMTqL/IO8tZwi6A9IgUkwf3bIqwTgQUoFX3SchaJL5zymBgNb8YcrbI3e?=
 =?us-ascii?Q?hpVh5Wlc1NKPXbMsES4RnHXvrfsfz9udEuayc3bEPzcY3zaOvBadDw91sInc?=
 =?us-ascii?Q?RnIAK1LMbgVqOqfn9gPTss6jTc25W+ZZJKJbbnU9s90ZrE1hzJdn3mtmCVV1?=
 =?us-ascii?Q?qiICimBEN1s+VgEKNU4A4l+d3aXcBPV/C+Uob/D3ueaL0AmDaIe7StYtW0k6?=
 =?us-ascii?Q?TJR7D8K7YmXuRfYSlPZQz5XDJMtSXzzGliIacW3c32fVSvU10fEE45uSfx84?=
 =?us-ascii?Q?mh5BiK5gqPtNLfIdS5VpQZAbNG/stveSRBYipb1gQ7r5EAPs/14DuixtlU1P?=
 =?us-ascii?Q?Kbiwas2fiHG9XdQxpH0hrhtQt3/mTLoJpyqWHqg7bbb46itm32BU3lqnhw+y?=
 =?us-ascii?Q?WP/gTj6n3tM/Y2yJopABHZgzVrV0XIy6pFkvs56DoSBSMldAL/jaJKTvBwa+?=
 =?us-ascii?Q?gIS//0IWhnP11HP0J7JHzR3Oru8lEW0scflhG//uOOmgkCZsz3c5B6upyMUM?=
 =?us-ascii?Q?Sos3iHUJbMC11SqUMg7vTXB4eZ4CKrgXDW5cZ4UR6Sz6coZC9v7Sr+/mtxic?=
 =?us-ascii?Q?O+K/NtmM4qMutuPclrQC9OFI9u0tpvLSLN/Xf9evI8DHbLkItMFCHX3xrdkI?=
 =?us-ascii?Q?XJBeVzgeMnNGmlU8oGdmyTCW?=
X-OriginatorOrg: amd.com
X-MS-Exchange-CrossTenant-Network-Message-Id: b15a3250-dd1f-4b49-06fb-08d976d1d31a
X-MS-Exchange-CrossTenant-AuthSource: SA0PR12MB4430.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 13 Sep 2021 16:16:21.2940
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 3dd8961f-e488-4e60-8e11-a82d994e183d
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: mQdOScBOU/mAg4BEre3gT7P2TfD90DabV+g3JBWRAhFtQTSpIJV4YNUkvBNrH5SeymqXqXHMrfiTl8vIweozcw==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: SA0PR12MB4349
Precedence: bulk
List-ID: <linux-ext4.vger.kernel.org>
X-Mailing-List: linux-ext4@vger.kernel.org

When CPU is connected throug XGMI, it has coherent
access to VRAM resource. In this case that resource
is taken from a table in the device gmc aperture base.
This resource is used along with the device type, which could
be DEVICE_PRIVATE or DEVICE_PUBLIC to create the device
page map region.

Signed-off-by: Alex Sierra <alex.sierra@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
---
v7:
Remove lookup_resource call, so export symbol for this function
is not longer required. Patch dropped "kernel: resource:
lookup_resource as exported symbol"
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 32 +++++++++++++++---------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index ffad39ffa8c6..d0e04f79a06e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -866,7 +866,7 @@ int svm_migrate_init(struct amdgpu_device *adev)
 {
 	struct kfd_dev *kfddev = adev->kfd.dev;
 	struct dev_pagemap *pgmap;
-	struct resource *res;
+	struct resource *res = NULL;
 	unsigned long size;
 	void *r;
 
@@ -881,22 +881,29 @@ int svm_migrate_init(struct amdgpu_device *adev)
 	 * should remove reserved size
 	 */
 	size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20);
-	res = devm_request_free_mem_region(adev->dev, &iomem_resource, size);
-	if (IS_ERR(res))
-		return -ENOMEM;
+	if (adev->gmc.xgmi.connected_to_cpu) {
+		pgmap->range.start = adev->gmc.aper_base;
+		pgmap->range.end = adev->gmc.aper_base + adev->gmc.aper_size - 1;
+		pgmap->type = MEMORY_DEVICE_PUBLIC;
+	} else {
+		res = devm_request_free_mem_region(adev->dev, &iomem_resource, size);
+		if (IS_ERR(res))
+			return -ENOMEM;
+		pgmap->range.start = res->start;
+		pgmap->range.end = res->end;
+		pgmap->type = MEMORY_DEVICE_PRIVATE;
+	}
 
-	pgmap->type = MEMORY_DEVICE_PRIVATE;
 	pgmap->nr_range = 1;
-	pgmap->range.start = res->start;
-	pgmap->range.end = res->end;
 	pgmap->ops = &svm_migrate_pgmap_ops;
 	pgmap->owner = SVM_ADEV_PGMAP_OWNER(adev);
-	pgmap->flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+	pgmap->flags = 0;
 	r = devm_memremap_pages(adev->dev, pgmap);
 	if (IS_ERR(r)) {
 		pr_err("failed to register HMM device memory\n");
-		devm_release_mem_region(adev->dev, res->start,
-					res->end - res->start + 1);
+		if (pgmap->type == MEMORY_DEVICE_PRIVATE)
+			devm_release_mem_region(adev->dev, res->start,
+						res->end - res->start + 1);
 		return PTR_ERR(r);
 	}
 
@@ -915,6 +922,7 @@ void svm_migrate_fini(struct amdgpu_device *adev)
 	struct dev_pagemap *pgmap = &adev->kfd.dev->pgmap;
 
 	devm_memunmap_pages(adev->dev, pgmap);
-	devm_release_mem_region(adev->dev, pgmap->range.start,
-				pgmap->range.end - pgmap->range.start + 1);
+	if (pgmap->type == MEMORY_DEVICE_PRIVATE)
+		devm_release_mem_region(adev->dev, pgmap->range.start,
+					pgmap->range.end - pgmap->range.start + 1);
 }
-- 
2.32.0