Received-SPF: pass (google.com: domain of linux-crypto+bounces-4733-linux.lists.archive=gmail.com@vger.kernel.org designates 139.178.88.99 as permitted sender) client-ip=139.178.88.99;
Message-ID: <d95d73d2-c728-4da2-b68e-d2e39a518ae6@oracle.com>
Date: Tue, 4 Jun 2024 14:47:27 -0700
User-Agent: Mozilla Thunderbird
Subject: Re: [PATCH v9 10/19] x86: Secure Launch SMP bringup support
To: Jarkko Sakkinen <jarkko@kernel.org>, linux-kernel@vger.kernel.org,
        x86@kernel.org, linux-integrity@vger.kernel.org,
        linux-doc@vger.kernel.org, linux-crypto@vger.kernel.org,
        kexec@lists.infradead.org, linux-efi@vger.kernel.org,
        iommu@lists.linux-foundation.org
Cc: dpsmith@apertussolutions.com, tglx@linutronix.de, mingo@redhat.com,
        bp@alien8.de, hpa@zytor.com, dave.hansen@linux.intel.com,
        ardb@kernel.org, mjg59@srcf.ucam.org,
        James.Bottomley@hansenpartnership.com, peterhuewe@gmx.de, jgg@ziepe.ca,
        luto@amacapital.net, nivedita@alum.mit.edu,
        herbert@gondor.apana.org.au, davem@davemloft.net, corbet@lwn.net,
        ebiederm@xmission.com, dwmw2@infradead.org, baolu.lu@linux.intel.com,
        kanth.ghatraju@oracle.com, andrew.cooper3@citrix.com,
        trenchboot-devel@googlegroups.com, ross.philipson@oracle.com
References: <20240531010331.134441-1-ross.philipson@oracle.com>
 <20240531010331.134441-11-ross.philipson@oracle.com>
 <D1RI46IG5GSA.17H7M0DIQGRQ0@kernel.org>
Content-Language: en-US
From: ross.philipson@oracle.com
In-Reply-To: <D1RI46IG5GSA.17H7M0DIQGRQ0@kernel.org>
Content-Type: text/plain; charset=UTF-8; format=flowed
Content-Transfer-Encoding: 7bit
Precedence: bulk
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: 
	=?utf-8?B?WnJiMjNBSy9SS294TklvVmdkbXJEenozY3N0YlZ2ZnI3MXU0U3IwMXYxRWJp?=
 =?utf-8?B?OUJ0S1BiQ21pMzhPOUlWWG9ueENNVFAzMElPNGxHOGp6NDBkRTBxWVZKSGZ5?=
 =?utf-8?B?SHorYTRVSGxWSnY3T1ZLVk5kenBtT3l1QlVZczZ1bnBnMkNwazNCc3doQzd3?=
 =?utf-8?B?YTJZRndKYjh1U3gwY08zNVp5elhWUFhTRmplQXo5WkVsN3BXWGs3TXdxUXg0?=
 =?utf-8?B?RlY2RGNmaUFrYWZOZlY5VzJjTWNVRXJkd3FBVkdZZG9NOS9nRDU2RUdxZjZR?=
 =?utf-8?B?aCtwdkFxYWpTVDJOZEFwM3I3L0pIN3lIM3U5Y084UUZnZjhxN2FicW9taUdP?=
 =?utf-8?B?NGkvdWdEeVNhNkwzN013VWdROXRYdkhodHhEUEIzM29yV01DcHZDVFp3QlZZ?=
 =?utf-8?B?MVQ4eHZGUjNXUHZvTTVUZVU1YUxIdnppSGZFRGFVMWl6d0dVd0ViUENYSE5v?=
 =?utf-8?B?Tjl4RDZWOEFmVG5CdU5WZXEySWp3Y2ZtWkF0UTI4Mk5LbHB3bHZ1Z1g3VFMr?=
 =?utf-8?B?SnJ6V1VZQm1vNjNTTG9ZV1hwTTh0dk5iRjdYbXo3bzBhdThDNE9uTWhxbGh4?=
 =?utf-8?B?V3NGb2dNVzg5dHpNbWJHMEtlU3ppUVdNN2k0bnZIK1BOc2hYUUMvVXI5dEQ5?=
 =?utf-8?B?WVhPdkFRWElSZkdzQjdjN2tYRWRvZHhZYktPR09XK0pKNFl6L0pmWS9Vbm5t?=
 =?utf-8?B?bWFNYWhiVEFwZCtkWUxIdmtnamg1VmRZUURmYUd1dFhBVTloTCtHM3NQOHVQ?=
 =?utf-8?B?WU5MWTlTTURtSHRWZ3pjUWNpbkdDNm53MnBuMWlOTS9TOS9GVktVa1hLNUo2?=
 =?utf-8?B?eFIyZERvWC85RFdObGtoNTBQblM5aW1rOGRQdUk2b1FyQXJoRys1Mi8ydDRl?=
 =?utf-8?B?UkRuT1dpNk5Pa1VLdlpZSVBLQUZrUWlXMnlnWDFVY3lRL0pFaHpJM2NmNzlx?=
 =?utf-8?B?Z0J6R1VGWHFvRWk1Q0pDOW5VN01UR0I1VUw3SktVRjBKT2puR1R0UURTNXgy?=
 =?utf-8?B?bFNPLzA0encvTURXQXAyLzNwT3JWdkpkWFJoTW5FLzBZOHBaU0IzblcwZEtO?=
 =?utf-8?B?Zk9wTHkzQ0NtYzdxNTFNUWJjUWR2azBCNm1xMUVxZmJJT2RJUFFSQjVKeXZW?=
 =?utf-8?B?ZnJmZXozaUhyYmJVN3lneTFVSVdCVEdCTkxvdVBKWGkwQ2EyNHJSemVGVmRU?=
 =?utf-8?B?eWZzRzNrWG5iRUJwTWhFdFNnTHV4SlZHalhXOWk2UDl5Ky9iNkN6SXFkSktp?=
 =?utf-8?B?SDNhUWZCY1V4MGZsTncwK1FLZnd1TFFMY3VnS0ZiL28yOHNCZ0FtU1R1OGxE?=
 =?utf-8?B?UmFtN2N5WCtGZnZXWmg5NGcrMEl3K0Y5R2RrZUJLMVFoNVF5RGJPRUNsN3hq?=
 =?utf-8?B?S3MxQ2dBc0dINnllVWpJRXhSN1BCWXB6MHpEK2ZLSWl6ancya3ZQVmVidCtD?=
 =?utf-8?B?dW9vWFhKSSs3Ykp2U0ZoZ05NL2NaTjhabnBRbzZQMEhZNXZSL0gyWlBxeEtJ?=
 =?utf-8?B?bUdac3hDQXdvWFRSRTRMU2liTExTZlZ0STZWN2NKcmw4SjJXUXpMZDBLVkZh?=
 =?utf-8?B?a2VPZmtNME15MjRWcGJIY1UvMFQremJqOCtGSTVoM1NHT2lGelRFRDFlcmdB?=
 =?utf-8?B?ck1ZUW1QV1JUdW9FZmJodHdrTTVmRGF1MUk2Y1ZCbkx2cXRSL1RJSVFTekVX?=
 =?utf-8?B?eEVxTitIR0NycHhQSU1qV1pZQVZsMUtrSUpaR2FyVzQwTkJWU0ZWbmFDTDZQ?=
 =?utf-8?B?SDJ4ZUsvekFIZWFZUXFKVFh4QUg1Q0x1YjhlcUdYdkxFdjJmWnJ2WE0zcHYx?=
 =?utf-8?B?S2Jja3BuZ0tMYjhoUUI0cUhjMW5URTRrbWZXYStUYnRzRkVVYmVOV3gyOXdO?=
 =?utf-8?B?VXRoWlB3NStGYmRCS2NJb2dnWGtZSG1Yc0xiT1FPVyt0NitXdEpudDhvcWpq?=
 =?utf-8?B?REc1WVkyemMzVGhFL1hTVmU5ZnFWUEkwTGllQW5lSlpvLzhRVmFtYTRCKzVU?=
 =?utf-8?B?Vk0ycmlheXg4NWJzOXE1L1Bvcjg3WlRmN1Zvek1lQ1BXRFhkTXI2SXl6WURy?=
 =?utf-8?B?WDBsdE56S3lHOVh5OWlFQWg3dkNkd21zRndCUSt6NGx1RlFxb0RpLzFqQ0pV?=
 =?utf-8?B?cjA4eWpzQWhzelFVOEZ2Y2FFbEU1eExCVkxKajdjYXd0TTZzck1qUnEwOHZR?=
 =?utf-8?B?Q1E9PQ==?=
X-MS-Exchange-AntiSpam-ExternalHop-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-ExternalHop-MessageData-0: 
	cClLsQyAGgkOYXzomJwuPL7pW7cMqm9blNuEY8ypoowlHbVjia8n0NVadJ7drxf8wDv3TVRLqqP1F9huSzJXoSqiRv1J+X17D19Zl/ti63/r7SuoG+sprrPBNIEqEVVFUjvJ1DXcAFmJOnDm059O5b5nyk/tO+HOFuuBbf35tYWF9mnr/OLjYNjg798zjaYakoTAri4YUicW3uHCpiwiFHPWrAOpHv1RBeJfo40tss7wPPgficX/xXUfNCbEbvLyGNTGq44XRFeCt+NjtN4EcAJo8wny7WShUuvYj92Wa7LK3SXS99hRd/1vn3t2WIZWKzOBpRDLsVNMOfV9vya28D56gfYatCb8bkVwEOeqCO7G0UlOL5DbPGb+FMlw1m6aw5kJF5JYR8pi8UD4mQ8mXvXPQeSl3X0tKhsbt5UdQqYKc35kFyFpgx46g+HzraHaobufVEgXKKsZN5LNCHiHfF9OPqDFWN6cS/44YmVzVRyBxXJ/zHTLDZh8KlimW4LBcsilXmcgy8oFHohtimOe3oUiHGZU4eG4hqq7TbVwWkY4QHzo/BTm9EAGZoxaAHxH3BwBZ+FWywkWv2v5qCt851DBmM9RxxkRhGS8SLsHtBA=
X-OriginatorOrg: oracle.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 5989da17-c904-452d-6cfb-08dc84dff136
X-MS-Exchange-CrossTenant-AuthSource: DS0PR10MB7224.namprd10.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 04 Jun 2024 21:47:34.0974
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 4e2c6054-71cb-48f1-bd6c-3a9705aca71b
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: Og0TDfYNGhWhc09KJInkG224Yr5FEK0yOgnAmXT05aJR3ILOT+O7m7ahPdznT6GUrR06++QMegjVkf8j9FXp4w1Xo6k+RFC8YRkNIGZHf2U=
X-MS-Exchange-Transport-CrossTenantHeadersStamped: CH2PR10MB4183
X-Proofpoint-Virus-Version: vendor=baseguard
 engine=ICAP:2.0.293,Aquarius:18.0.1039,Hydra:6.0.680,FMLib:17.12.28.16
 definitions=2024-06-04_11,2024-06-04_02,2024-05-17_01
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 mlxscore=0 bulkscore=0 spamscore=0
 malwarescore=0 suspectscore=0 phishscore=0 mlxlogscore=999 adultscore=0
 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2405010000
 definitions=main-2406040175
X-Proofpoint-ORIG-GUID: J5WPSzH-kcvCWoa7VGBswKltqcWqX7WF
X-Proofpoint-GUID: J5WPSzH-kcvCWoa7VGBswKltqcWqX7WF

On 6/4/24 1:05 PM, Jarkko Sakkinen wrote:
> On Fri May 31, 2024 at 4:03 AM EEST, Ross Philipson wrote:
>> On Intel, the APs are left in a well documented state after TXT performs
>> the late launch. Specifically they cannot have #INIT asserted on them so
>> a standard startup via INIT/SIPI/SIPI cannot be performed. Instead the
>> early SL stub code uses MONITOR and MWAIT to park the APs. The realmode/init.c
>> code updates the jump address for the waiting APs with the location of the
>> Secure Launch entry point in the RM piggy after it is loaded and fixed up.
>> As the APs are woken up by writing the monitor, the APs jump to the Secure
>> Launch entry point in the RM piggy which mimics what the real mode code would
>> do then jumps to the standard RM piggy protected mode entry point.
>>
>> Signed-off-by: Ross Philipson <ross.philipson@oracle.com>
>> ---
>>   arch/x86/include/asm/realmode.h      |  3 ++
>>   arch/x86/kernel/smpboot.c            | 58 +++++++++++++++++++++++++++-
>>   arch/x86/realmode/init.c             |  3 ++
>>   arch/x86/realmode/rm/header.S        |  3 ++
>>   arch/x86/realmode/rm/trampoline_64.S | 32 +++++++++++++++
>>   5 files changed, 97 insertions(+), 2 deletions(-)
>>
>> diff --git a/arch/x86/include/asm/realmode.h b/arch/x86/include/asm/realmode.h
>> index 87e5482acd0d..339b48e2543d 100644
>> --- a/arch/x86/include/asm/realmode.h
>> +++ b/arch/x86/include/asm/realmode.h
>> @@ -38,6 +38,9 @@ struct real_mode_header {
>>   #ifdef CONFIG_X86_64
>>   	u32	machine_real_restart_seg;
>>   #endif
>> +#ifdef CONFIG_SECURE_LAUNCH
>> +	u32	sl_trampoline_start32;
>> +#endif
>>   };
>>   
>>   /* This must match data at realmode/rm/trampoline_{32,64}.S */
>> diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
>> index 0c35207320cb..adb521221d6c 100644
>> --- a/arch/x86/kernel/smpboot.c
>> +++ b/arch/x86/kernel/smpboot.c
>> @@ -60,6 +60,7 @@
>>   #include <linux/stackprotector.h>
>>   #include <linux/cpuhotplug.h>
>>   #include <linux/mc146818rtc.h>
>> +#include <linux/slaunch.h>
>>   
>>   #include <asm/acpi.h>
>>   #include <asm/cacheinfo.h>
>> @@ -868,6 +869,56 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle)
>>   	return 0;
>>   }
>>   
>> +#ifdef CONFIG_SECURE_LAUNCH
>> +
>> +static bool slaunch_is_txt_launch(void)
>> +{
>> +	if ((slaunch_get_flags() & (SL_FLAG_ACTIVE|SL_FLAG_ARCH_TXT)) ==
>> +	    (SL_FLAG_ACTIVE | SL_FLAG_ARCH_TXT))
>> +		return true;
>> +
>> +	return false;
>> +}
> 
> static inline bool slaunch_is_txt_launch(void)
> {
> 	u32 mask =  SL_FLAG_ACTIVE | SL_FLAG_ARCH_TXT;
> 
> 	return slaunch_get_flags() & mask == mask;
> }

Actually I think I can take your suggested change and move this function 
to the main header files since this check is done elsewhere. And later I 
can make others like slaunch_is_skinit_launch(). Thanks.

> 
> 
>> +
>> +/*
>> + * TXT AP startup is quite different than normal. The APs cannot have #INIT
>> + * asserted on them or receive SIPIs. The early Secure Launch code has parked
>> + * the APs using monitor/mwait. This will wake the APs by writing the monitor
>> + * and have them jump to the protected mode code in the rmpiggy where the rest
>> + * of the SMP boot of the AP will proceed normally.
>> + */
>> +static void slaunch_wakeup_cpu_from_txt(int cpu, int apicid)
>> +{
>> +	struct sl_ap_wake_info *ap_wake_info;
>> +	struct sl_ap_stack_and_monitor *stack_monitor = NULL;
> 
> struct sl_ap_stack_and_monitor *stack_monitor; /* note: no initialization */
> struct sl_ap_wake_info *ap_wake_info;

Will fix.

> 
> 
>> +
>> +	ap_wake_info = slaunch_get_ap_wake_info();
>> +
>> +	stack_monitor = (struct sl_ap_stack_and_monitor *)__va(ap_wake_info->ap_wake_block +
>> +							       ap_wake_info->ap_stacks_offset);
>> +
>> +	for (unsigned int i = TXT_MAX_CPUS - 1; i >= 0; i--) {
>> +		if (stack_monitor[i].apicid == apicid) {
>> +			/* Write the monitor */
> 
> I'd remove this comment.

Sure.

Ross

> 
>> +			stack_monitor[i].monitor = 1;
>> +			break;
>> +		}
>> +	}
>> +}
>> +
>> +#else
>> +
>> +static inline bool slaunch_is_txt_launch(void)
>> +{
>> +	return false;
>> +}
>> +
>> +static inline void slaunch_wakeup_cpu_from_txt(int cpu, int apicid)
>> +{
>> +}
>> +
>> +#endif  /* !CONFIG_SECURE_LAUNCH */
>> +
>>   /*
>>    * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
>>    * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
>> @@ -877,7 +928,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle)
>>   static int do_boot_cpu(u32 apicid, int cpu, struct task_struct *idle)
>>   {
>>   	unsigned long start_ip = real_mode_header->trampoline_start;
>> -	int ret;
>> +	int ret = 0;
>>   
>>   #ifdef CONFIG_X86_64
>>   	/* If 64-bit wakeup method exists, use the 64-bit mode trampoline IP */
>> @@ -922,12 +973,15 @@ static int do_boot_cpu(u32 apicid, int cpu, struct task_struct *idle)
>>   
>>   	/*
>>   	 * Wake up a CPU in difference cases:
>> +	 * - Intel TXT DRTM launch uses its own method to wake the APs
>>   	 * - Use a method from the APIC driver if one defined, with wakeup
>>   	 *   straight to 64-bit mode preferred over wakeup to RM.
>>   	 * Otherwise,
>>   	 * - Use an INIT boot APIC message
>>   	 */
>> -	if (apic->wakeup_secondary_cpu_64)
>> +	if (slaunch_is_txt_launch())
>> +		slaunch_wakeup_cpu_from_txt(cpu, apicid);
>> +	else if (apic->wakeup_secondary_cpu_64)
>>   		ret = apic->wakeup_secondary_cpu_64(apicid, start_ip);
>>   	else if (apic->wakeup_secondary_cpu)
>>   		ret = apic->wakeup_secondary_cpu(apicid, start_ip);
>> diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
>> index f9bc444a3064..d95776cb30d3 100644
>> --- a/arch/x86/realmode/init.c
>> +++ b/arch/x86/realmode/init.c
>> @@ -4,6 +4,7 @@
>>   #include <linux/memblock.h>
>>   #include <linux/cc_platform.h>
>>   #include <linux/pgtable.h>
>> +#include <linux/slaunch.h>
>>   
>>   #include <asm/set_memory.h>
>>   #include <asm/realmode.h>
>> @@ -210,6 +211,8 @@ void __init init_real_mode(void)
>>   
>>   	setup_real_mode();
>>   	set_real_mode_permissions();
>> +
>> +	slaunch_fixup_jump_vector();
>>   }
>>   
>>   static int __init do_init_real_mode(void)
>> diff --git a/arch/x86/realmode/rm/header.S b/arch/x86/realmode/rm/header.S
>> index 2eb62be6d256..3b5cbcbbfc90 100644
>> --- a/arch/x86/realmode/rm/header.S
>> +++ b/arch/x86/realmode/rm/header.S
>> @@ -37,6 +37,9 @@ SYM_DATA_START(real_mode_header)
>>   #ifdef CONFIG_X86_64
>>   	.long	__KERNEL32_CS
>>   #endif
>> +#ifdef CONFIG_SECURE_LAUNCH
>> +	.long	pa_sl_trampoline_start32
>> +#endif
>>   SYM_DATA_END(real_mode_header)
>>   
>>   	/* End signature, used to verify integrity */
>> diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S
>> index 14d9c7daf90f..b0ce6205d7ea 100644
>> --- a/arch/x86/realmode/rm/trampoline_64.S
>> +++ b/arch/x86/realmode/rm/trampoline_64.S
>> @@ -122,6 +122,38 @@ SYM_CODE_END(sev_es_trampoline_start)
>>   
>>   	.section ".text32","ax"
>>   	.code32
>> +#ifdef CONFIG_SECURE_LAUNCH
>> +	.balign 4
>> +SYM_CODE_START(sl_trampoline_start32)
>> +	/*
>> +	 * The early secure launch stub AP wakeup code has taken care of all
>> +	 * the vagaries of launching out of TXT. This bit just mimics what the
>> +	 * 16b entry code does and jumps off to the real startup_32.
>> +	 */
>> +	cli
>> +	wbinvd
>> +
>> +	/*
>> +	 * The %ebx provided is not terribly useful since it is the physical
>> +	 * address of tb_trampoline_start and not the base of the image.
>> +	 * Use pa_real_mode_base, which is fixed up, to get a run time
>> +	 * base register to use for offsets to location that do not have
>> +	 * pa_ symbols.
>> +	 */
>> +	movl    $pa_real_mode_base, %ebx
>> +
>> +	LOCK_AND_LOAD_REALMODE_ESP lock_pa=1
>> +
>> +	lgdt    tr_gdt(%ebx)
>> +	lidt    tr_idt(%ebx)
>> +
>> +	movw	$__KERNEL_DS, %dx	# Data segment descriptor
>> +
>> +	/* Jump to where the 16b code would have jumped */
>> +	ljmpl	$__KERNEL32_CS, $pa_startup_32
>> +SYM_CODE_END(sl_trampoline_start32)
>> +#endif
>> +
>>   	.balign 4
>>   SYM_CODE_START(startup_32)
>>   	movl	%edx, %ss
> 
> BR, Jarkko
>