Received-SPF: pass (google.com: domain of linux-ext4-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
References: <20220505213438.25064-1-alex.sierra@amd.com>
 <20220505213438.25064-15-alex.sierra@amd.com>
User-agent: mu4e 1.6.9; emacs 27.1
From:   Alistair Popple <apopple@nvidia.com>
To:     Alex Sierra <alex.sierra@amd.com>
Cc:     jgg@nvidia.com, david@redhat.com, Felix.Kuehling@amd.com,
        linux-mm@kvack.org, rcampbell@nvidia.com,
        linux-ext4@vger.kernel.org, linux-xfs@vger.kernel.org,
        amd-gfx@lists.freedesktop.org, dri-devel@lists.freedesktop.org,
        hch@lst.de, jglisse@redhat.com, willy@infradead.org,
        akpm@linux-foundation.org
Subject: Re: [PATCH v1 14/15] tools: add hmm gup tests for device coherent type
Date:   Mon, 16 May 2022 18:02:56 +1000
In-reply-to: <20220505213438.25064-15-alex.sierra@amd.com>
Message-ID: <87ee0taokl.fsf@nvdebian.thelocal>
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?ZHweYjSS6llbStAtlU2NOx74UGsbnT9CZWQ00RCT4F8zrYHzEsj8noaoxWXZ?=
 =?us-ascii?Q?f0waaBYzdEgNJGy2SLkx9Ue+gg5KcCz5tPtMNJdFGwcYUAktDrf3mLgU1+SC?=
 =?us-ascii?Q?q3DZBF6vi1SlCmUXTYnFpJ/4yBRYP1UkP5zakbA5DbRSlSMk+No+RbuRhcac?=
 =?us-ascii?Q?nsKWK5rh/0p/+Gt+pELlGhMHaikAgH2cz+5rgchqXeM6k6evnmNFIRtFXqY3?=
 =?us-ascii?Q?e5+q5E0E2wnX6DMdr3sS9IEVz8WL31aSkwewUAOPfrH6QFUzRLv6ZOb3kNWc?=
 =?us-ascii?Q?Z/8Dlw1O7K6UknIw9fw7ByBNbChP7VIkr51AREndOZwTdRqdaSMAWydC3DKG?=
 =?us-ascii?Q?1CG5cT3ue1e7bwdk4YDgFvG+zZUyknQbYwI3K+L40zXFpkpKZfZHg1B/6uxt?=
 =?us-ascii?Q?cSdO/yZcHG3w7ceP+mSIHfSgD7/NiX6gFx/HnImo3UeYlDfmMwbarBVY5TBc?=
 =?us-ascii?Q?xrRbmPeavwEw6IaZ1Gy2SJYZG2obHzueRtAPEPnfWqnFU8EFzKVxDRhsKGI3?=
 =?us-ascii?Q?Cf1BGXI5UrMrkS710KPby/95VrLor98R3iHesPywkW8NEH2AxurMNdX4raSc?=
 =?us-ascii?Q?lvDTbF4wrFhgPsLRqGbI9kwaxjGJ0kxuMjQlaxJlB6Z9zR5ne7abbJEEyXCU?=
 =?us-ascii?Q?iAopwGuDOAJjVLXSAGNNmZ9UrMAxzz0MgcNjVD5sH7xodZsu1Wec9x4nTssa?=
 =?us-ascii?Q?LXO9gtDW2gV/Q3FqtWe1HPJDtbvSFHRLX3XfShIhiv/sGCFU+L8ZhDS95EM9?=
 =?us-ascii?Q?KWL0CGBNUOPChhfs26v3JyP+VhVbgnJy4HXkA8aNa2SHAfLfe/OgMV/Aoeia?=
 =?us-ascii?Q?fPGuL3NT6dPL9HjfAnmViDcMMlmOiRbLxNaxkM7mZuE86qaLjJUzDHHcta6t?=
 =?us-ascii?Q?MlCJCzIbXx/uYMhwvPQ3eEid42e1kIWnllENQ6AblOvgleh82Ib2qh/yqBUO?=
 =?us-ascii?Q?CqQFw6GRLmIyyrQvy/CCxvfEpdCIvbmBYGhlDMZH/yQhJDEBZT8YFLeORBSp?=
 =?us-ascii?Q?jtBtGtJM5T8LGGf/B1kwi8APs4+N0YAEpw1nskhRZMp3yITgtDMLyBL0mTGu?=
 =?us-ascii?Q?Q0GRYAhERz3ouMMICGEWxhv6Nt6K6V3a2aa/5fogDiFFp/rgXOsWoH2fJ+pC?=
 =?us-ascii?Q?G+U1jBRJCoNCq9X/4+ulFABtHsk5Qr8TCW0rcS5JfjjemSPgCHMSd1WJgkJR?=
 =?us-ascii?Q?Rn1lcdB9ekWHYWejPYUzpGgwPkfux82aN5Omqes+e4k1Yj9VVmF3p9N9SFgd?=
 =?us-ascii?Q?VLI+L+dn4oK7SZs3YAITY3NPcqsfL5LifvDNulsUVvPbAayRK93Aa4YgWCfr?=
 =?us-ascii?Q?MNiE6f7Y5iOhYnnFEmvjx5x9haf2skUoiKP/LDKQd42Bh5+ZGKUlBmOmxV/C?=
 =?us-ascii?Q?IxoMzMc5L+zM/zdWtA+u9OzvTZ1VwVyXGe6YDZfkHQ9Rz9xKN0oD46O4Pa6T?=
 =?us-ascii?Q?sEeH+UHlk3V520HdeScz4ch7NUBtwNsYeBgHBKdXZ5zWc6W21JMwZ+x1opqM?=
 =?us-ascii?Q?Z53iwFiE/LEUloiUDR7QcOrEr9qUQ72CBHDyCJsIbahAdfdx6HOZ+AbA2kGP?=
 =?us-ascii?Q?L5HZzxVDNsXw4BMjwutP2keIaH7KNu9Ypn5e991f8YzOZk7c0BjzZyjYhdKM?=
 =?us-ascii?Q?Zj3RxTFIgMjzWzB23DHyyNtEA9pV1JbCrewF7CYJkCEf9D9aC2Gy7HMMzgdw?=
 =?us-ascii?Q?5XHVUhibF6GLOlDIYOOjHiGrtJFXPK6iUwkgUnDUJL8KHJH7caQvD4uMsur8?=
 =?us-ascii?Q?rdoVguywaA=3D=3D?=
X-OriginatorOrg: Nvidia.com
X-MS-Exchange-CrossTenant-Network-Message-Id: b3511d01-2899-43e7-46af-08da37130a81
X-MS-Exchange-CrossTenant-AuthSource: BYAPR12MB3176.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 16 May 2022 08:06:54.7903
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: sW5dNvIl1u7K/xdnn7m6PfaYycZBwBkbIAeH4KpabPZ0NSG+VVGPwwBziFaW1X3hmiV9Y0ntILv1zhh/vIaCeA==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BYAPR12MB2936
X-Spam-Status: No, score=-2.0 required=5.0 tests=BAYES_00,DKIMWL_WL_HIGH,
        DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,FORGED_SPF_HELO,
        RCVD_IN_DNSWL_NONE,RCVD_IN_MSPIKE_H2,SPF_HELO_PASS,SPF_NONE,
        T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on
        lindbergh.monkeyblade.net
Precedence: bulk
List-ID: <linux-ext4.vger.kernel.org>
X-Mailing-List: linux-ext4@vger.kernel.org


Alex Sierra <alex.sierra@amd.com> writes:

> The intention is to test hmm device coherent type under different get
> user pages paths. Also, test gup with FOLL_LONGTERM flag set in
> device coherent pages. These pages should get migrated back to system
> memory.
>
> Signed-off-by: Alex Sierra <alex.sierra@amd.com>
> ---
>  tools/testing/selftests/vm/hmm-tests.c | 104 +++++++++++++++++++++++++
>  1 file changed, 104 insertions(+)
>
> diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c
> index 84ec8c4a1dc7..65e30ab6494c 100644
> --- a/tools/testing/selftests/vm/hmm-tests.c
> +++ b/tools/testing/selftests/vm/hmm-tests.c
> @@ -36,6 +36,7 @@
>   * in the usual include/uapi/... directory.
>   */
>  #include "../../../../lib/test_hmm_uapi.h"
> +#include "../../../../mm/gup_test.h"
>
>  struct hmm_buffer {
>  	void		*ptr;
> @@ -60,6 +61,8 @@ enum {
>  #define NTIMES		10
>
>  #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
> +/* Just the flags we need, copied from mm.h: */
> +#define FOLL_WRITE	0x01	/* check pte is writable */
>
>  FIXTURE(hmm)
>  {
> @@ -1766,4 +1769,105 @@ TEST_F(hmm, exclusive_cow)
>  	hmm_buffer_free(buffer);
>  }
>
> +static int gup_test_exec(int gup_fd, unsigned long addr,
> +			 int cmd, int npages, int size)
> +{
> +	struct gup_test gup = {
> +		.nr_pages_per_call	= npages,
> +		.addr			= addr,
> +		.gup_flags		= FOLL_WRITE,
> +		.size			= size,
> +	};
> +
> +	if (ioctl(gup_fd, cmd, &gup)) {
> +		perror("ioctl on error\n");
> +		return errno;
> +	}
> +
> +	return 0;
> +}
> +
> +/*
> + * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
> + * This should trigger a migration back to system memory for both, private
> + * and coherent type pages.
> + * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
> + * to your configuration before you run it.
> + */
> +TEST_F(hmm, hmm_gup_test)
> +{
> +	struct hmm_buffer *buffer;
> +	int gup_fd;
> +	unsigned long npages;
> +	unsigned long size;
> +	unsigned long i;
> +	int *ptr;
> +	int ret;
> +	unsigned char *m;
> +
> +	gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
> +	if (gup_fd == -1)
> +		SKIP(return, "Skipping test, could not find gup_test driver");
> +
> +	npages = 3;
> +	size = npages << self->page_shift;
> +
> +	buffer = malloc(sizeof(*buffer));
> +	ASSERT_NE(buffer, NULL);
> +
> +	buffer->fd = -1;
> +	buffer->size = size;
> +	buffer->mirror = malloc(size);
> +	ASSERT_NE(buffer->mirror, NULL);
> +
> +	buffer->ptr = mmap(NULL, size,
> +			   PROT_READ | PROT_WRITE,
> +			   MAP_PRIVATE | MAP_ANONYMOUS,
> +			   buffer->fd, 0);
> +	ASSERT_NE(buffer->ptr, MAP_FAILED);
> +
> +	/* Initialize buffer in system memory. */
> +	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
> +		ptr[i] = i;
> +
> +	/* Migrate memory to device. */
> +	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
> +	ASSERT_EQ(ret, 0);
> +	ASSERT_EQ(buffer->cpages, npages);
> +	/* Check what the device read. */
> +	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
> +		ASSERT_EQ(ptr[i], i);
> +
> +	ASSERT_EQ(gup_test_exec(gup_fd,
> +				(unsigned long)buffer->ptr,
> +				GUP_BASIC_TEST, 1, self->page_size), 0);
> +	ASSERT_EQ(gup_test_exec(gup_fd,
> +				(unsigned long)buffer->ptr + 1 * self->page_size,
> +				GUP_FAST_BENCHMARK, 1, self->page_size), 0);
> +	ASSERT_EQ(gup_test_exec(gup_fd,
> +				(unsigned long)buffer->ptr + 2 * self->page_size,
> +				PIN_LONGTERM_BENCHMARK, 1, self->page_size), 0);
> +
> +	/* Take snapshot to CPU pagetables */
> +	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
> +	ASSERT_EQ(ret, 0);
> +	ASSERT_EQ(buffer->cpages, npages);
> +	m = buffer->mirror;
> +	if (hmm_is_coherent_type(variant->device_number)) {
> +		ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
> +		ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
> +	} else {
> +		ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
> +		ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
> +	}
> +	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
> +	/* Check again the content on the pages. Make sure there's no

Very minor nit-pick, but if you happen to be re-spinning anyway the
comment formatting needs fixing and a new line before the comment would
be nice. Otherwise:

Reviewed-by: Alistair Popple <apopple@nvidia.com>

> +	 * corrupted data.
> +	 */
> +	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
> +		ASSERT_EQ(ptr[i], i);
> +
> +	close(gup_fd);
> +	hmm_buffer_free(buffer);
> +}
>  TEST_HARNESS_MAIN