Subject: Re: [PATCH v3] xen/balloon: Mark unallocated host memory as UNUSABLE
To: Boris Ostrovsky <boris.ostrovsky@oracle.com>, xen-devel@lists.xen.org,
        linux-kernel@vger.kernel.org
Cc: jgross@suse.com, helgaas@kernel.org, JBeulich@suse.com
References: <1513778746-6155-1-git-send-email-boris.ostrovsky@oracle.com>
From: =?UTF-8?Q?Christian_K=c3=b6nig?= <christian.koenig@amd.com>
Message-ID: <c02af663-df02-17bb-1a89-b5f069f7f535@amd.com>
Date: Sun, 24 Dec 2017 10:29:12 +0100
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101
 Thunderbird/52.5.0
MIME-Version: 1.0
In-Reply-To: <1513778746-6155-1-git-send-email-boris.ostrovsky@oracle.com>
Content-Type: text/plain; charset=utf-8; format=flowed
Content-Transfer-Encoding: 8bit
Content-Language: en-US
SpamDiagnosticOutput: 1:99
SpamDiagnosticMetadata: NSPM
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 24 Dec 2017 09:29:23.2793 (UTC)
X-MS-Exchange-CrossTenant-Network-Message-Id: d9e6c347-2df4-4f11-589d-08d54ab0d248
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 3dd8961f-e488-4e60-8e11-a82d994e183d
X-MS-Exchange-Transport-CrossTenantHeadersStamped: CY4PR12MB1301
Sender: linux-kernel-owner@vger.kernel.org
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org
Content-Length: 8718
Lines: 270

Am 20.12.2017 um 15:05 schrieb Boris Ostrovsky:
> Commit f5775e0b6116 ("x86/xen: discard RAM regions above the maximum
> reservation") left host memory not assigned to dom0 as available for
> memory hotplug.
>
> Unfortunately this also meant that those regions could be used by
> others. Specifically, commit fa564ad96366 ("x86/PCI: Enable a 64bit BAR
> on AMD Family 15h (Models 00-1f, 30-3f, 60-7f)") may try to map those
> addresses as MMIO.
>
> To prevent this mark unallocated host memory as E820_TYPE_UNUSABLE (thus
> effectively reverting f5775e0b6116) and keep track of that region as
> a hostmem resource that can be used for the hotplug.
>
> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>

Acked-by: Christian König <christian.koenig@amd.com>

> ---
> Changes in v3:
> * Use PFN_PHYS
> * Replace kzalloc with kmalloc
> * Declare arch_xen_balloon_init prototype in balloon.h
> * Rename resources (s/memory/RAM/)
> * Clarify (I think) comment when populating hostmem_resource
> * Print open-ended interval on insert_resource() error
> * Constify declaration of struct e820_entry *entry
>
>   arch/x86/xen/enlighten.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++++
>   arch/x86/xen/setup.c     |  6 ++--
>   drivers/xen/balloon.c    | 65 ++++++++++++++++++++++++++++++++------
>   include/xen/balloon.h    |  5 +++
>   4 files changed, 144 insertions(+), 13 deletions(-)
>
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index d669e9d..c9081c6 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -1,8 +1,12 @@
> +#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
> +#include <linux/bootmem.h>
> +#endif
>   #include <linux/cpu.h>
>   #include <linux/kexec.h>
>   
>   #include <xen/features.h>
>   #include <xen/page.h>
> +#include <xen/interface/memory.h>
>   
>   #include <asm/xen/hypercall.h>
>   #include <asm/xen/hypervisor.h>
> @@ -331,3 +335,80 @@ void xen_arch_unregister_cpu(int num)
>   }
>   EXPORT_SYMBOL(xen_arch_unregister_cpu);
>   #endif
> +
> +#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
> +void __init arch_xen_balloon_init(struct resource *hostmem_resource)
> +{
> +	struct xen_memory_map memmap;
> +	int rc;
> +	unsigned int i, last_guest_ram;
> +	phys_addr_t max_addr = PFN_PHYS(max_pfn);
> +	struct e820_table *xen_e820_table;
> +	const struct e820_entry *entry;
> +	struct resource *res;
> +
> +	if (!xen_initial_domain())
> +		return;
> +
> +	xen_e820_table = kmalloc(sizeof(*xen_e820_table), GFP_KERNEL);
> +	if (!xen_e820_table)
> +		return;
> +
> +	memmap.nr_entries = ARRAY_SIZE(xen_e820_table->entries);
> +	set_xen_guest_handle(memmap.buffer, xen_e820_table->entries);
> +	rc = HYPERVISOR_memory_op(XENMEM_machine_memory_map, &memmap);
> +	if (rc) {
> +		pr_warn("%s: Can't read host e820 (%d)\n", __func__, rc);
> +		goto out;
> +	}
> +
> +	last_guest_ram = 0;
> +	for (i = 0; i < memmap.nr_entries; i++) {
> +		if (xen_e820_table->entries[i].addr >= max_addr)
> +			break;
> +		if (xen_e820_table->entries[i].type == E820_TYPE_RAM)
> +			last_guest_ram = i;
> +	}
> +
> +	entry = &xen_e820_table->entries[last_guest_ram];
> +	if (max_addr >= entry->addr + entry->size)
> +		goto out; /* No unallocated host RAM. */
> +
> +	hostmem_resource->start = max_addr;
> +	hostmem_resource->end = entry->addr + entry->size;
> +
> +	/*
> +	 * Mark non-RAM regions between the end of dom0 RAM and end of host RAM
> +	 * as unavailable. The rest of that region can be used for hotplug-based
> +	 * ballooning.
> +	 */
> +	for (; i < memmap.nr_entries; i++) {
> +		entry = &xen_e820_table->entries[i];
> +
> +		if (entry->type == E820_TYPE_RAM)
> +			continue;
> +
> +		if (entry->addr >= hostmem_resource->end)
> +			break;
> +
> +		res = kzalloc(sizeof(*res), GFP_KERNEL);
> +		if (!res)
> +			goto out;
> +
> +		res->name = "Unavailable host RAM";
> +		res->start = entry->addr;
> +		res->end = (entry->addr + entry->size < hostmem_resource->end) ?
> +			    entry->addr + entry->size : hostmem_resource->end;
> +		rc = insert_resource(hostmem_resource, res);
> +		if (rc) {
> +			pr_warn("%s: Can't insert [%llx - %llx) (%d)\n",
> +				__func__, res->start, res->end, rc);
> +			kfree(res);
> +			goto  out;
> +		}
> +	}
> +
> + out:
> +	kfree(xen_e820_table);
> +}
> +#endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
> diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
> index c114ca7..6e0d208 100644
> --- a/arch/x86/xen/setup.c
> +++ b/arch/x86/xen/setup.c
> @@ -808,7 +808,6 @@ char * __init xen_memory_setup(void)
>   	addr = xen_e820_table.entries[0].addr;
>   	size = xen_e820_table.entries[0].size;
>   	while (i < xen_e820_table.nr_entries) {
> -		bool discard = false;
>   
>   		chunk_size = size;
>   		type = xen_e820_table.entries[i].type;
> @@ -824,11 +823,10 @@ char * __init xen_memory_setup(void)
>   				xen_add_extra_mem(pfn_s, n_pfns);
>   				xen_max_p2m_pfn = pfn_s + n_pfns;
>   			} else
> -				discard = true;
> +				type = E820_TYPE_UNUSABLE;
>   		}
>   
> -		if (!discard)
> -			xen_align_and_add_e820_region(addr, chunk_size, type);
> +		xen_align_and_add_e820_region(addr, chunk_size, type);
>   
>   		addr += chunk_size;
>   		size -= chunk_size;
> diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
> index f77e499..065f0b6 100644
> --- a/drivers/xen/balloon.c
> +++ b/drivers/xen/balloon.c
> @@ -257,10 +257,25 @@ static void release_memory_resource(struct resource *resource)
>   	kfree(resource);
>   }
>   
> +/*
> + * Host memory not allocated to dom0. We can use this range for hotplug-based
> + * ballooning.
> + *
> + * It's a type-less resource. Setting IORESOURCE_MEM will make resource
> + * management algorithms (arch_remove_reservations()) look into guest e820,
> + * which we don't want.
> + */
> +static struct resource hostmem_resource = {
> +	.name   = "Host RAM",
> +};
> +
> +void __attribute__((weak)) __init arch_xen_balloon_init(struct resource *res)
> +{}
> +
>   static struct resource *additional_memory_resource(phys_addr_t size)
>   {
> -	struct resource *res;
> -	int ret;
> +	struct resource *res, *res_hostmem;
> +	int ret = -ENOMEM;
>   
>   	res = kzalloc(sizeof(*res), GFP_KERNEL);
>   	if (!res)
> @@ -269,13 +284,42 @@ static struct resource *additional_memory_resource(phys_addr_t size)
>   	res->name = "System RAM";
>   	res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
>   
> -	ret = allocate_resource(&iomem_resource, res,
> -				size, 0, -1,
> -				PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
> -	if (ret < 0) {
> -		pr_err("Cannot allocate new System RAM resource\n");
> -		kfree(res);
> -		return NULL;
> +	res_hostmem = kzalloc(sizeof(*res), GFP_KERNEL);
> +	if (res_hostmem) {
> +		/* Try to grab a range from hostmem */
> +		res_hostmem->name = "Host memory";
> +		ret = allocate_resource(&hostmem_resource, res_hostmem,
> +					size, 0, -1,
> +					PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
> +	}
> +
> +	if (!ret) {
> +		/*
> +		 * Insert this resource into iomem. Because hostmem_resource
> +		 * tracks portion of guest e820 marked as UNUSABLE noone else
> +		 * should try to use it.
> +		 */
> +		res->start = res_hostmem->start;
> +		res->end = res_hostmem->end;
> +		ret = insert_resource(&iomem_resource, res);
> +		if (ret < 0) {
> +			pr_err("Can't insert iomem_resource [%llx - %llx]\n",
> +				res->start, res->end);
> +			release_memory_resource(res_hostmem);
> +			res_hostmem = NULL;
> +			res->start = res->end = 0;
> +		}
> +	}
> +
> +	if (ret) {
> +		ret = allocate_resource(&iomem_resource, res,
> +					size, 0, -1,
> +					PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
> +		if (ret < 0) {
> +			pr_err("Cannot allocate new System RAM resource\n");
> +			kfree(res);
> +			return NULL;
> +		}
>   	}
>   
>   #ifdef CONFIG_SPARSEMEM
> @@ -287,6 +331,7 @@ static struct resource *additional_memory_resource(phys_addr_t size)
>   			pr_err("New System RAM resource outside addressable RAM (%lu > %lu)\n",
>   			       pfn, limit);
>   			release_memory_resource(res);
> +			release_memory_resource(res_hostmem);
>   			return NULL;
>   		}
>   	}
> @@ -765,6 +810,8 @@ static int __init balloon_init(void)
>   	set_online_page_callback(&xen_online_page);
>   	register_memory_notifier(&xen_memory_nb);
>   	register_sysctl_table(xen_root);
> +
> +	arch_xen_balloon_init(&hostmem_resource);
>   #endif
>   
>   #ifdef CONFIG_XEN_PV
> diff --git a/include/xen/balloon.h b/include/xen/balloon.h
> index 4914b93..61f410f 100644
> --- a/include/xen/balloon.h
> +++ b/include/xen/balloon.h
> @@ -44,3 +44,8 @@ static inline void xen_balloon_init(void)
>   {
>   }
>   #endif
> +
> +#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
> +struct resource;
> +void arch_xen_balloon_init(struct resource *hostmem_resource);
> +#endif