SGkgTGludXMsDQoNClRoZSBmb2xsb3dpbmcgY2hhbmdlcyBzaW5jZSBjb21taXQgNzg2YjcxZjVi
NzU0MjczY2NlZjZkOTQ2MmU1MjA2MmIzZTFmOTg3NzoNCg0KICBNZXJnZSB0YWcgJ25kczMyLWZv
ci1saW51cy00LjE3LWZpeGVzJyBvZiBnaXQ6Ly9naXQua2VybmVsLm9yZy9wdWIvc2NtL2xpbnV4
L2tlcm5lbC9naXQvZ3JlZW50aW1lL2xpbnV4ICgyMDE4LTA1LTI4IDA1OjI1OjU3IC0wNzAwKQ0K
DQphcmUgYXZhaWxhYmxlIGluIHRoZSBHaXQgcmVwb3NpdG9yeSBhdDoNCg0KICBnaXQ6Ly9naXQu
bGludXgtbmZzLm9yZy9wcm9qZWN0cy90cm9uZG15L2xpbnV4LW5mcy5naXQgdGFncy9uZnMtZm9y
LTQuMTgtMQ0KDQpmb3IgeW91IHRvIGZldGNoIGNoYW5nZXMgdXAgdG8gOTNiN2Y3YWQyMDE4ZDIw
Mzc1NTliMWQwODkyNDE3ODY0Yzc4YjM3MToNCg0KICBza2lwIExBWU9VVFJFVFVSTiBpZiBsYXlv
dXQgaXMgaW52YWxpZCAoMjAxOC0wNi0xMiAwODo0ODowNCAtMDQwMCkNCg0KQ2hlZXJzLA0KICBU
cm9uZA0KDQotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tDQpORlMgY2xpZW50IHVwZGF0ZXMgZm9yIExpbnV4IDQuMTgNCg0KSGln
aGxpZ2h0cyBpbmNsdWRlOg0KDQpTdGFibGUgZml4ZXM6DQotIEZpeCBhIDEtYnl0ZSBzdGFjayBv
dmVyZmxvdyBpbiBuZnNfaWRtYXBfcmVhZF9hbmRfdmVyaWZ5X21lc3NhZ2UNCi0gRml4IGEgaGFu
ZyBkdWUgdG8gaW5jb3JyZWN0IGVycm9yIHJldHVybnMgaW4gcnBjcmRtYV9jb252ZXJ0X2lvdnMo
KQ0KLSBSZXZlcnQgYW4gaW5jb3JyZWN0IGNoYW5nZSB0byB0aGUgTkZTdjQuMSBjYWxsYmFjayBj
aGFubmVsDQotIEZpeCBhIGJ1ZyBpbiB0aGUgTkZTdjQuMSBzZXF1ZW5jZSBlcnJvciBoYW5kbGlu
Zw0KDQpGZWF0dXJlcyBhbmQgb3B0aW1pc2F0aW9uczoNCi0gU3VwcG9ydCBmb3IgcGlnZ3liYWNr
aW5nIGEgTEFZT1VUR0VUIG9wZXJhdGlvbiB0byB0aGUgT1BFTiBjb21wb3VuZA0KLSBSRE1BIHBl
cmZvcm1hbmNlIGVuaGFuY2VtZW50cyB0byBkZWFsIHdpdGggdHJhbnNwb3J0IGNvbmdlc3Rpb24N
Ci0gQWRkIHByb3BlciBTUERYIHRhZ3MgZm9yIE5ldEFwcC1jb250cmlidXRlZCBSRE1BIHNvdXJj
ZQ0KLSBEbyBub3QgcmVxdWVzdCBkZWxlZ2F0ZWQgZmlsZSBhdHRyaWJ1dGVzIChzaXplK2NoYW5n
ZSkgZnJvbSB0aGUgc2VydmVyDQotIE9wdGltaXNlIGF3YXkgYSBHRVRBVFRSIGluIHRoZSBsb29r
dXAgcmV2YWxpZGF0ZSBjb2RlIHdoZW4gZG9pbmcgTkZTdjQgT1BFTg0KLSBPcHRpbWlzZSBhd2F5
IHVubmVjZXNzYXJ5IGxvb2t1cHMgZm9yIHJlbmFtZSB0YXJnZXRzDQotIE1pc2MgcGVyZm9ybWFu
Y2UgaW1wcm92ZW1lbnRzIHdoZW4gZnJlZWluZyBORlN2NCBkZWxlZ2F0aW9ucw0KDQpCdWdmaXhl
cyBhbmQgY2xlYW51cHM6DQotIFRyeSB0byBmYWlsIHF1aWNrbHkgaWYgcHJvdG89cmRtYQ0KLSBD
bGVhbiB1cCBSRE1BIHJlY2VpdmUgdHJhY2UgcG9pbnRzDQotIEZpeCBzaWxseXJlbmFtZSB0byBy
ZXR1cm4gdGhlIGRlbGVnYXRpb24gd2hlbiBhcHByb3ByaWF0ZQ0KLSBNaXNjIGF0dHJpYnV0ZSBy
ZXZhbGlkYXRpb24gZml4ZXMNCi0gSW1tZWRpYXRlbHkgY2xlYXIgdGhlIHBORlMgbGF5b3V0IG9u
IGEgZmlsZSB3aGVuIHRoZSBzZXJ2ZXIgcmV0dXJucyBFU1RBTEUNCi0gUmV0dXJuIE5GUzRFUlJf
REVMQVkgd2hlbiBkZWxlZ2F0aW9uL2xheW91dCByZWNhbGxzIGZhaWwgZHVlIHRvIGlncmFiKCkN
Ci0gRml4IHRoZSBjbGllbnQgYmVoYXZpb3VyIG9uIE5GUzRFUlJfU0VRX0ZBTFNFX1JFVFJZDQoN
Ci0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0NCkFubmEgU2NodW1ha2VyICg0KToNCiAgICAgIE5GUzogTW92ZSBjYWxsIHRvIG5m
czRfc3RhdGVfcHJvdGVjdF93cml0ZSgpIHRvIG5mczRfd3JpdGVfc2V0dXAoKQ0KICAgICAgTkZT
OiBNb3ZlIGNhbGwgdG8gbmZzNF9zdGF0ZV9wcm90ZWN0KCkgdG8gbmZzNF9jb21taXRfc2V0dXAo
KQ0KICAgICAgTkZTOiBQYXNzICJwcml2aWxlZ2VkIiB2YWx1ZSB0byBuZnM0X2luaXRfc2VxdWVu
Y2UoKQ0KICAgICAgTkZTOiBNZXJnZSBuZnM0MV9mcmVlX3N0YXRlaWQoKSB3aXRoIF9uZnM0MV9m
cmVlX3N0YXRlaWQoKQ0KDQpCZW5qYW1pbiBDb2RkaW5ndG9uICgxKToNCiAgICAgIE5GU3Y0OiBE
b24ndCBhZGQgYSBuZXcgbG9jayBvbiBhbiBpbnRlcnJ1cHRlZCB3YWl0IGZvciBMT0NLDQoNCkNo
dWNrIExldmVyICgyMSk6DQogICAgICB4cHJ0cmRtYTogQWRkIHByb3BlciBTUERYIHRhZ3MgZm9y
IE5ldEFwcC1jb250cmlidXRlZCBzb3VyY2UNCiAgICAgIHhwcnRyZG1hOiBUcnkgdG8gZmFpbCBx
dWlja2x5IGlmIHByb3RvPXJkbWENCiAgICAgIHhwcnRyZG1hOiBDcmVhdGUgdHJhbnNwb3J0J3Mg
Q00gSUQgaW4gdGhlIGNvcnJlY3QgbmV0d29yayBuYW1lc3BhY2UNCiAgICAgIHhwcnRyZG1hOiBG
aXggbWF4X3NlbmRfd3IgY29tcHV0YXRpb24NCiAgICAgIFNVTlJQQzogSW5pdGlhbGl6ZSBycGNf
cnFzdCBvdXRzaWRlIG9mIHhwcnQtPnJlc2VydmVfbG9jaw0KICAgICAgU1VOUlBDOiBBZGQgYSAt
PmZyZWVfc2xvdCB0cmFuc3BvcnQgY2FsbG91dA0KICAgICAgeHBydHJkbWE6IEludHJvZHVjZSAt
PmFsbG9jX3Nsb3QgY2FsbC1vdXQgZm9yIHhwcnRyZG1hDQogICAgICB4cHJ0cmRtYTogTWFrZSBy
cGNfcnFzdCBwYXJ0IG9mIHJwY3JkbWFfcmVxDQogICAgICB4cHJ0cmRtYTogQ2xlYW4gdXAgUmVj
ZWl2ZSB0cmFjZSBwb2ludHMNCiAgICAgIHhwcnRyZG1hOiBNb3ZlIFJlY2VpdmUgcG9zdGluZyB0
byBSZWNlaXZlIGhhbmRsZXINCiAgICAgIHhwcnRyZG1hOiBSZW1vdmUgcnBjcmRtYV9lcF97cG9z
dF9yZWN2LCBwb3N0X2V4dHJhX3JlY3Z9DQogICAgICB4cHJ0cmRtYTogUmVtb3ZlIHJwY3JkbWFf
YnVmZmVyX2dldF9yZXFfbG9ja2VkKCkNCiAgICAgIHhwcnRyZG1hOiBSZW1vdmUgcnBjcmRtYV9i
dWZmZXJfZ2V0X3JlcF9sb2NrZWQoKQ0KICAgICAgeHBydHJkbWE6IE1ha2UgcnBjcmRtYV9zZW5k
Y3R4X3B1dF9sb2NrZWQoKSBhIHN0YXRpYyBmdW5jdGlvbg0KICAgICAgeHBydHJkbWE6IFJldHVy
biAtRU5PQlVGUyB3aGVuIG5vIHBhZ2VzIGFyZSBhdmFpbGFibGUNCiAgICAgIHhwcnRyZG1hOiBN
b3ZlIGNvbW1vbiB3YWl0X2Zvcl9idWZmZXJfc3BhY2UgY2FsbCB0byBwYXJlbnQgZnVuY3Rpb24N
CiAgICAgIHhwcnRyZG1hOiBXYWl0IG9uIGVtcHR5IHNlbmRjdHggcXVldWUNCiAgICAgIHhwcnRy
ZG1hOiBBZGQgdHJhY2VfeHBydHJkbWFfZG1hX21hcChtcikNCiAgICAgIHhwcnRyZG1hOiBSZW1v
dmUgdHJhbnNmZXJ0eXBlcyBhcnJheQ0KICAgICAgTkZTdjQuMDogUmVtb3ZlIGNsX2lwYWRkciBm
cm9tIG5vbi1VQ1MgY2xpZW50IElEDQogICAgICBORlN2NC4wOiBSZW1vdmUgdHJhbnNwb3J0IHBy
b3RvY29sIG5hbWUgZnJvbSBub24tVUNTIGNsaWVudCBJRA0KDQpEYXZlIFd5c29jaGFuc2tpICgx
KToNCiAgICAgIE5GU3Y0OiBGaXggcG9zc2libGUgMS1ieXRlIHN0YWNrIG92ZXJmbG93IGluIG5m
c19pZG1hcF9yZWFkX2FuZF92ZXJpZnlfbWVzc2FnZQ0KDQpGcmVkIElzYW1hbiAoMTQpOg0KICAg
ICAgcG5mczogUmVtb3ZlIHJlZHVuZGFudCBhc3NpZ25tZW50IGZyb20gbmZzNF9wcm9jX2xheW91
dGdldCgpLg0KICAgICAgcG5mczogU3RvcmUgcmV0dXJuIHZhbHVlIG9mIGRlY29kZV9sYXlvdXRn
ZXQgZm9yIGxhdGVyIHByb2Nlc3NpbmcNCiAgICAgIE5GUzQ6IG1vdmUgY3R4IGludG8gbmZzNF9y
dW5fb3Blbl90YXNrDQogICAgICBwbmZzOiBBZGQgbGF5b3V0IGRyaXZlciBmbGFnIFBORlNfTEFZ
T1VUR0VUX09OX09QRU4NCiAgICAgIHBuZnM6IHJlZmFjdG9yIHNlbmRfbGF5b3V0Z2V0DQogICAg
ICBwbmZzOiBtb3ZlIGFsbG9jYXRpb25zIG91dCBvZiBuZnM0X3Byb2NfbGF5b3V0Z2V0DQogICAg
ICBwbmZzOiBBZGQgY29uZGl0aW9uYWwgZW5jb2RlL2RlY29kZSBvZiBMQVlPVVRHRVQgd2l0aGlu
IE9QRU4gY29tcG91bmQNCiAgICAgIHBuZnM6IE1vdmUgbmZzNF9vcGVuZGF0YSBpbnRvIG5mczRf
ZnMuaA0KICAgICAgcG5mczogQ2hhbmdlIHBuZnNfYWxsb2NfaW5pdF9sYXlvdXRnZXRfYXJncyBj
YWxsIHNpZ25hdHVyZQ0KICAgICAgcG5mczogQWRkIExBWU9VVEdFVCB0byBPUEVOIG9mIGEgbmV3
IGZpbGUNCiAgICAgIHBuZnM6IEFkZCBMQVlPVVRHRVQgdG8gT1BFTiBvZiBhbiBleGlzdGluZyBm
aWxlDQogICAgICBwbmZzOiBTdG9wIGF0dGVtcHRpbmcgTEFZT1VUR0VUIG9uIE9QRU4gb24gZmFp
bHVyZQ0KICAgICAgcG5mczogQWRkIGJhcnJpZXIgdG8gcHJldmVudCBsZ29wZW4gdXNpbmcgTEFZ
T1VUR0VUIGR1cmluZyByZWNhbGwNCiAgICAgIHBuZnM6IEZpeCBtYW5pcHVsYXRpb24gb2YgTkZT
X0xBWU9VVF9GSVJTVF9MQVlPVVRHRVQNCg0KTmVpbEJyb3duICg0KToNCiAgICAgIE5GUzogc2xp
Z2h0IG9wdGltaXphdGlvbiBmb3Igd2Fsa2luZyBsaXN0IGZvciBkZWxlZ2F0aW9ucw0KICAgICAg
TkZTOiB1c2UgY29uZF9yZXNjaGVkKCkgd2hlbiByZXN0YXJ0aW5nIHdhbGsgb2YgZGVsZWdhdGlv
biBsaXN0Lg0KICAgICAgcmN1bGlzdDogYWRkIGxpc3RfZm9yX2VhY2hfZW50cnlfZnJvbV9yY3Uo
KQ0KICAgICAgTkZTOiBBdm9pZCBxdWFkcmF0aWMgc2VhcmNoIHdoZW4gZnJlZWluZyBkZWxlZ2F0
aW9ucy4NCg0KT2xnYSBLb3JuaWV2c2thaWEgKDEpOg0KICAgICAgc2tpcCBMQVlPVVRSRVRVUk4g
aWYgbGF5b3V0IGlzIGludmFsaWQNCg0KVHJvbmQgTXlrbGVidXN0ICgzNSk6DQogICAgICBORlM6
IE9wdGltaXNlIGF3YXkgdGhlIGNsb3NlLXRvLW9wZW4gR0VUQVRUUiB3aGVuIHdlIGhhdmUgTkZT
djQgT1BFTg0KICAgICAgTkZTOiBJZiB0aGUgVkZTIHNldHMgTE9PS1VQX1JFVkFMIHRoZW4gZm9y
Y2UgYSBsb29rdXAgb2YgdGhlIGRlbnRyeQ0KICAgICAgTkZTOiBPcHRpbWlzZSBhd2F5IGxvb2t1
cHMgZm9yIHJlbmFtZSB0YXJnZXRzDQogICAgICBORlN2NDogT25seSBwYXNzIHRoZSBkZWxlZ2F0
aW9uIHRvIHNldGF0dHIgaWYgd2UncmUgc2VuZGluZyBhIHRydW5jYXRlDQogICAgICBORlN2NDog
Rml4IHNpbGx5cmVuYW1lIHRvIHJldHVybiB0aGUgZGVsZWdhdGlvbiB3aGVuIGFwcHJvcHJpYXRl
DQogICAgICBORlM6IEZpeCB1cCBzaWxseXJlbmFtZSgpDQogICAgICBORlM6IFNldCB0aGUgZm9y
Y2UgcmV2YWxpZGF0ZSBmbGFnIGlmIHRoZSBpbm9kZSBpcyBub3QgY29tcGxldGVseSBpbml0aWFs
aXNlZA0KICAgICAgTkZTOiBFbnN1cmUgd2UgcmV2YWxpZGF0ZSB0aGUgaW5vZGUgY29ycmVjdGx5
IGFmdGVyIHJlbW92ZSBvciByZW5hbWUNCiAgICAgIE5GUzogRW5zdXJlIHdlIHJldmFsaWRhdGUg
dGhlIGlub2RlIGNvcnJlY3RseSBhZnRlciBzZXRhY2wNCiAgICAgIE5GUzogRml4IHVwIG5mc19w
b3N0X29wX3VwZGF0ZV9pbm9kZSgpIHRvIGZvcmNlIGN0aW1lIHVwZGF0ZXMNCiAgICAgIE5GU3Y0
OiBBbHdheXMgY2xlYXIgdGhlIHBORlMgbGF5b3V0IHdoZW4gaGFuZGxpbmcgRVNUQUxFDQogICAg
ICBwTkZTOiBSZWZhY3RvciBuZnM0X2xheW91dGdldF9yZWxlYXNlKCkNCiAgICAgIE5GU3Y0L3Bu
ZnM6IEVuc3VyZSBwbmZzX3BhcnNlX2xnb3BlbigpIHdvbid0IHRyeSB0byBwYXJzZSB1bmluaXRp
YWxpc2VkIGRhdGENCiAgICAgIE5GU3Y0L3BuZnM6IERvbid0IHN3aXRjaCBvZmYgbGF5b3V0Z2V0
LW9uLW9wZW4gZm9yIHRyYW5zaWVudCBlcnJvcnMNCiAgICAgIHBORlM6IERvbid0IHNlbmQgTEFZ
T1VUR0VUIG9uIE9QRU4gZm9yIHJlYWQsIGlmIHdlIGFscmVhZHkgaGF2ZSBjYWNoZWQgZGF0YQ0K
ICAgICAgcG5mczogRG9uJ3QgY2FsbCBjb21taXQgb24gZmFpbGVkIGxheW91dGdldC1vbi1vcGVu
DQogICAgICBwbmZzOiBEb24ndCByZWxlYXNlIHRoZSBzZXF1ZW5jZSBzbG90IHVudGlsIHdlJ3Zl
IHByb2Nlc3NlZCBsYXlvdXRnZXQgb24gb3Blbg0KICAgICAgTkZTdjQ6IERvbid0IHJlcXVlc3Qg
c2l6ZStjaGFuZ2UgYXR0cmlidXRlIGlmIHRoZXkgYXJlIGRlbGVnYXRlZCB0byB1cw0KICAgICAg
TkZTOiBQYXNzIHRoZSBpbm9kZSBkb3duIHRvIHRoZSBnZXRhdHRyKCkgY2FsbGJhY2sNCiAgICAg
IE5GU3Y0OiBEb24ndCBhc2sgZm9yIGRlbGVnYXRlZCBhdHRyaWJ1dGVzIHdoZW4gcmV2YWxpZGF0
aW5nIHRoZSBpbm9kZQ0KICAgICAgTkZTdjQ6IERvbid0IGFzayBmb3IgZGVsZWdhdGVkIGF0dHJp
YnV0ZXMgd2hlbiBhZGRpbmcgYSBoYXJkIGxpbmsNCiAgICAgIE5GU3Y0OiBJZ25vcmUgTkZTX0lO
T19SRVZBTF9GT1JDRUQgaW4gbmZzNF9wcm9jX2FjY2Vzcw0KICAgICAgTkZTdjQ6IEVuc3VyZSB0
aGUgaW5vZGUgaXMgY2xlYW4gd2hlbiB3ZSBzZXQgYSBkZWxlZ2F0aW9uDQogICAgICBORlM6IGZp
eCB1cCBuZnNfc2V0YXR0cl91cGRhdGVfaW5vZGUNCiAgICAgIE5GUzogRml4IGF0dHJpYnV0ZSBy
ZXZhbGlkYXRpb24NCiAgICAgIE5GUzogSW1wcm92ZSBjYWNoaW5nIHdoaWxlIGhvbGRpbmcgYSBk
ZWxlZ2F0aW9uDQogICAgICBORlM6IElnbm9yZSBORlNfSU5PX1JFVkFMX0ZPUkNFRCBpbiBuZnNf
Y2hlY2tfaW5vZGVfYXR0cmlidXRlcygpDQogICAgICBORlM6IEZpbHRlciBjYWNoZSBpbnZhbGlk
YXRpb24gd2hlbiBob2xkaW5nIGEgZGVsZWdhdGlvbg0KICAgICAgTWVyZ2UgdGFnICduZnMtcmRt
YS1mb3ItNC4xOC0xJyBvZiBnaXQ6Ly9naXQubGludXgtbmZzLm9yZy9wcm9qZWN0cy9hbm5hL2xp
bnV4LW5mcw0KICAgICAgTkZTdjQ6IEZpeCBhIGNvbXBpbGVyIHdhcm5pbmcgd2hlbiBDT05GSUdf
TkZTX1Y0XzEgaXMgdW5kZWZpbmVkDQogICAgICBORlN2NDogUmV0dXJuIE5GUzRFUlJfREVMQVkg
d2hlbiBhIGRlbGVnYXRpb24gcmVjYWxsIGZhaWxzIGR1ZSB0byBpZ3JhYigpDQogICAgICBORlN2
NDogUmV0dXJuIE5GUzRFUlJfREVMQVkgd2hlbiBhIGxheW91dCByZWNhbGwgZmFpbHMgZHVlIHRv
IGlncmFiKCkNCiAgICAgIE5GU3Y0OiBSZXZlcnQgY29tbWl0IDVmODNkODZjZjUzMWQgKCJORlN2
NC54OiBGaXggd3JhcGFyb3VuZCBpc3N1ZXMuLiIpDQogICAgICBORlN2NDogRml4IGEgdHlwbyBp
biBuZnM0MV9zZXF1ZW5jZV9wcm9jZXNzDQogICAgICBORlN2NC4xOiBGaXggdGhlIGNsaWVudCBi
ZWhhdmlvdXIgb24gTkZTNEVSUl9TRVFfRkFMU0VfUkVUUlkNCg0KIGZzL25mcy9jYWxsYmFja19w
cm9jLmMgICAgICAgICAgICAgICAgICAgICB8ICA0MyArKy0tDQogZnMvbmZzL2NsaWVudC5jICAg
ICAgICAgICAgICAgICAgICAgICAgICAgIHwgICAzICstDQogZnMvbmZzL2RlbGVnYXRpb24uYyAg
ICAgICAgICAgICAgICAgICAgICAgIHwgIDg2ICsrKysrLS0NCiBmcy9uZnMvZGlyLmMgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgfCAgNTEgKysrLQ0KIGZzL25mcy9leHBvcnQuYyAgICAg
ICAgICAgICAgICAgICAgICAgICAgICB8ICAgMiArLQ0KIGZzL25mcy9mbGV4ZmlsZWxheW91dC9m
bGV4ZmlsZWxheW91dC5jICAgICB8ICAgMSArDQogZnMvbmZzL2lub2RlLmMgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgIHwgMTI2ICsrKysrKystLS0NCiBmcy9uZnMvbmZzM3Byb2MuYyAgICAg
ICAgICAgICAgICAgICAgICAgICAgfCAgMTMgKy0NCiBmcy9uZnMvbmZzNDJwcm9jLmMgICAgICAg
ICAgICAgICAgICAgICAgICAgfCAgIDYgKy0NCiBmcy9uZnMvbmZzNF9mcy5oICAgICAgICAgICAg
ICAgICAgICAgICAgICAgfCAgMjcgKy0NCiBmcy9uZnMvbmZzNGlkbWFwLmMgICAgICAgICAgICAg
ICAgICAgICAgICAgfCAgIDUgKy0NCiBmcy9uZnMvbmZzNHByb2MuYyAgICAgICAgICAgICAgICAg
ICAgICAgICAgfCAzOTEgKysrKysrKysrKysrKy0tLS0tLS0tLS0tLS0tLS0NCiBmcy9uZnMvbmZz
NHN0YXRlLmMgICAgICAgICAgICAgICAgICAgICAgICAgfCAgIDggKw0KIGZzL25mcy9uZnM0eGRy
LmMgICAgICAgICAgICAgICAgICAgICAgICAgICB8ICA2NSArKysrLQ0KIGZzL25mcy9wbmZzLmMg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICB8IDMzMSArKysrKysrKysrKysrKysrKysrKyst
LS0NCiBmcy9uZnMvcG5mcy5oICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfCAgMjggKyst
DQogZnMvbmZzL3Byb2MuYyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHwgIDEzICstDQog
ZnMvbmZzL3VubGluay5jICAgICAgICAgICAgICAgICAgICAgICAgICAgIHwgIDIwICstDQogZnMv
bmZzL3dyaXRlLmMgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHwgIDEwICstDQogaW5jbHVk
ZS9saW51eC9uZnNfZnNfc2IuaCAgICAgICAgICAgICAgICAgIHwgICAyICsNCiBpbmNsdWRlL2xp
bnV4L25mc194ZHIuaCAgICAgICAgICAgICAgICAgICAgfCAgMTUgKy0NCiBpbmNsdWRlL2xpbnV4
L3JjdWxpc3QuaCAgICAgICAgICAgICAgICAgICAgfCAgMTMgKw0KIGluY2x1ZGUvbGludXgvc3Vu
cnBjL3JwY19yZG1hLmggICAgICAgICAgICB8ICAgMSArDQogaW5jbHVkZS9saW51eC9zdW5ycGMv
eHBydC5oICAgICAgICAgICAgICAgIHwgICA2ICstDQogaW5jbHVkZS9saW51eC9zdW5ycGMveHBy
dHJkbWEuaCAgICAgICAgICAgIHwgICAxICsNCiBpbmNsdWRlL3RyYWNlL2V2ZW50cy9ycGNyZG1h
LmggICAgICAgICAgICAgfCAgNzYgKysrKy0tDQogbmV0L3N1bnJwYy9jbG50LmMgICAgICAgICAg
ICAgICAgICAgICAgICAgIHwgICAxICsNCiBuZXQvc3VucnBjL3hwcnQuYyAgICAgICAgICAgICAg
ICAgICAgICAgICAgfCAgMTcgKy0NCiBuZXQvc3VucnBjL3hwcnRyZG1hL2JhY2tjaGFubmVsLmMg
ICAgICAgICAgfCAxMDUgKysrLS0tLS0NCiBuZXQvc3VucnBjL3hwcnRyZG1hL2Ztcl9vcHMuYyAg
ICAgICAgICAgICAgfCAgMjMgKysNCiBuZXQvc3VucnBjL3hwcnRyZG1hL2Zyd3Jfb3BzLmMgICAg
ICAgICAgICAgfCAgMzEgKystDQogbmV0L3N1bnJwYy94cHJ0cmRtYS9tb2R1bGUuYyAgICAgICAg
ICAgICAgIHwgICAxICsNCiBuZXQvc3VucnBjL3hwcnRyZG1hL3JwY19yZG1hLmMgICAgICAgICAg
ICAgfCAgNjYgKystLS0NCiBuZXQvc3VucnBjL3hwcnRyZG1hL3N2Y19yZG1hX2JhY2tjaGFubmVs
LmMgfCAgIDEgKw0KIG5ldC9zdW5ycGMveHBydHJkbWEvdHJhbnNwb3J0LmMgICAgICAgICAgICB8
ICA2NCArKystLQ0KIG5ldC9zdW5ycGMveHBydHJkbWEvdmVyYnMuYyAgICAgICAgICAgICAgICB8
IDI5MSArKysrKysrKystLS0tLS0tLS0tLS0NCiBuZXQvc3VucnBjL3hwcnRyZG1hL3hwcnRfcmRt
YS5oICAgICAgICAgICAgfCAgMjYgKy0NCiBuZXQvc3VucnBjL3hwcnRzb2NrLmMgICAgICAgICAg
ICAgICAgICAgICAgfCAgIDQgKw0KIDM4IGZpbGVzIGNoYW5nZWQsIDEyNDAgaW5zZXJ0aW9ucygr
KSwgNzMzIGRlbGV0aW9ucygtKQ0KLS0gDQpUcm9uZCBNeWtsZWJ1c3QNCkxpbnV4IE5GUyBjbGll
bnQgbWFpbnRhaW5lciwgSGFtbWVyc3BhY2UNCnRyb25kLm15a2xlYnVzdEBoYW1tZXJzcGFjZS5j
b20NCg0K
On Tue, Jun 12, 2018 at 6:39 AM Trond Myklebust <[email protected]> wrote:
>
> NeilBrown (4):
> rculist: add list_for_each_entry_from_rcu()
Oh christ, people. Not another one of these.
We need to start having a real rule in place where people DO NOT PLAY
GAMES WITH RCU LISTS.
Adding Paul McKenney to the list, since he clearly wasn't for the
actual development, and the commit has no sign that it was sanity
checked.
This whole "let's re-start RCU walking in the middle after we've
dropped the RCU read lock" needs a hell of a lot more thought than
people seem to actually be giving it.
Maybe the code is actually correct - it looks ok to me. But every time
I see it I just shudder.
What people actually want is to just have a cursor entry in an RCU
list. I'd much rather have *that*, than have people make up these
insane ad-hoc cursor entries that are insane and have horrible
semantics.
For example, looking at that commit e04bbf6b1bbe ("NFS: Avoid
quadratic search when freeing delegations"), it even talks about the
ad-hoc cursor entry not actually being reliable, and how that's ok
because the code doesn't care. No, random odd behavior that is
basically never ever going to be testable is *NOT* ok.
So could we please have a cursor entry for RCU walking, and actually
have a agreed-upon way to do this? Maybe even using the low bit in the
"next" field to mark a cursor entry generically - the same way we
already do for the HEAD field in the bit-locked lists, just on the
actual entries _on_ the list instead.
Because we now have *two* of these odd special "restart the loop in
the middle" come in during the same merge window. That really implies
to me that we should have a _proper_ solution for the problem, not
this horribly nasty case where people make up their own pseudo-cursors
that have odd and questionable semantics.
Paul, comments?
Linus
On Tue, Jun 12, 2018 at 10:42 AM Linus Torvalds
<[email protected]> wrote:
>
> So could we please have a cursor entry for RCU walking, and actually
> have a agreed-upon way to do this? Maybe even using the low bit in the
> "next" field to mark a cursor entry generically - the same way we
> already do for the HEAD field in the bit-locked lists, just on the
> actual entries _on_ the list instead.
The real problem with a RCU cursor is that the lifetime of the cursor
is also RCU extended, so you can't do the traditional "just allocate
the cursor entry on the stack" that you can with synchronous locked
lists.
That makes it slightly more inconvenient to do simple cursors.
The dcache code has a fairly clean "dcache cursor", but it does use a
whole "struct dentry" for it (and it depends on "next_positive()" to
just skip them because dursors are never _positive_ dentries, so
there's some subtlety there).
But at least it's a very explicit cursor model, not that odd
delegation inode that seems to have a life as a real inode too.
So maybe a generic rcu-list cursor is too hard to do sanely, but can
we at least encourage that very explicit cursor model that is *only* a
cursor, and might not be touched/moved/reallocated by something else?
Linus
Final note (for now) on this: I've merged the nfs code, but I really
am obviously not happy with these crazy random ad-hoc
cursor-not-cursor list games.
Linus
On Tue, Jun 12 2018, Linus Torvalds wrote:
> Final note (for now) on this: I've merged the nfs code, but I really
> am obviously not happy with these crazy random ad-hoc
> cursor-not-cursor list games.
>
> Linus
Hi Linus,
thanks for merging the code despite your reservations.
Yes, we could create a generic rcu-list cursor. I have given it some
thought but didn't like the added complexity. As there were existing
objects in the list that could be used as a cursor, that seemed to me to
be the better solution.
As you say, and cursor would need to be allocated from a slab, not on
the stack. We could use a SLAB_TYPESAFE_BY_RCU and not need to use rcu
to delay the freeing.
The lsb in the next pointer of the cursor would be 1 to indicate the
cursor.
Any iteration of the list would need to look out for this flag.
When found it would need to skip over any cursors to the next
non-cursor, then repeat the skip and make sure it found the same
non-cursor. This guards against the cursor moving while it is being
examined.
Any walk that needed to place a cursor would need to get an exclusive
lock on the list from the start. This is more locking overhead than
just grabbing the lock to optimistically take a reference on the
"current" item which I did in the NFS patch. If the lists were
normally short that might not be a problem. In this case the list can
get quite long so the extra locking might be noticeable.
Deleting objects from the list would need to be careful to preserve the
flag bit, but that is the least difficult part.
FYI I have an open proposal to improve the cursor used by rhashtable
for rhashtable_walk - it sometimes needs to drop out of RCU in the
middle of a bucket chain. In that case the chain is normally short (16 is
considered so long that the hash must have been compromised) and I
propose an insertion sort to keep the addresses of objects in numerical
order. This way the address of the last object found can work as a stable
cursor - we just search through the list until an object has a larger
address.
So my perspective is that while an rcu_cursor_list could be developed,
I'm not sure it would always (or ever?) be the best solution to a
given problem.
I can turn these thoughts into a patch if you like and see what people
think.
Thanks,
NeilBrown
On Tue, Jun 12, 2018 at 11:08:31AM -0700, Linus Torvalds wrote:
> Final note (for now) on this: I've merged the nfs code, but I really
> am obviously not happy with these crazy random ad-hoc
> cursor-not-cursor list games.
We did review this one, and Neil did make requested changes to the comment
header and documentation. However, as you say, I will push back harder
on future RCU list traversal macros.
Thanx, Paul
On Tue, Jun 12, 2018 at 6:02 PM Paul E. McKenney
<[email protected]> wrote:
>
> We did review this one, and Neil did make requested changes to the comment
> header and documentation.
Oh, I checked the commit for "acked-by" from you and didn't see any,
so I was assuming this was just Neil and Trond on their own..
Linus
On Tue, Jun 12, 2018 at 06:11:50PM -0700, Linus Torvalds wrote:
> On Tue, Jun 12, 2018 at 6:02 PM Paul E. McKenney
> <[email protected]> wrote:
> >
> > We did review this one, and Neil did make requested changes to the comment
> > header and documentation.
>
> Oh, I checked the commit for "acked-by" from you and didn't see any,
> so I was assuming this was just Neil and Trond on their own..
Hmmm... I gave them a Reviewed-by, but perhaps it got lost.
http://lkml.kernel.org/r/[email protected]
Thanx, Paul
On Tue, Jun 12 2018, Paul E. McKenney wrote:
> On Tue, Jun 12, 2018 at 06:11:50PM -0700, Linus Torvalds wrote:
>> On Tue, Jun 12, 2018 at 6:02 PM Paul E. McKenney
>> <[email protected]> wrote:
>> >
>> > We did review this one, and Neil did make requested changes to the comment
>> > header and documentation.
>>
>> Oh, I checked the commit for "acked-by" from you and didn't see any,
>> so I was assuming this was just Neil and Trond on their own..
>
> Hmmm... I gave them a Reviewed-by, but perhaps it got lost.
>
> http://lkml.kernel.org/r/[email protected]
>
> Thanx, Paul
Sorry about that. I think I planned to add it after I heard back from
Trond what he thought of the patch, but I didn't hear anything until it
landed. I'll try to be more proactive next time.
Thanks,
NeilBrown
On Wed, Jun 13, 2018 at 11:42:51AM +1000, NeilBrown wrote:
> On Tue, Jun 12 2018, Paul E. McKenney wrote:
>
> > On Tue, Jun 12, 2018 at 06:11:50PM -0700, Linus Torvalds wrote:
> >> On Tue, Jun 12, 2018 at 6:02 PM Paul E. McKenney
> >> <[email protected]> wrote:
> >> >
> >> > We did review this one, and Neil did make requested changes to the comment
> >> > header and documentation.
> >>
> >> Oh, I checked the commit for "acked-by" from you and didn't see any,
> >> so I was assuming this was just Neil and Trond on their own..
> >
> > Hmmm... I gave them a Reviewed-by, but perhaps it got lost.
> >
> > http://lkml.kernel.org/r/[email protected]
>
> Sorry about that. I think I planned to add it after I heard back from
> Trond what he thought of the patch, but I didn't hear anything until it
> landed. I'll try to be more proactive next time.
Not a problem from my viewpoint. There may come a time when I need to
care about contribution metrics, but I don't need to at the moment. ;-)
Thanx, Paul
Unfortunately the patch for adding list_for_each_entry_from_rcu()
wasn't the final patch after all review. It is functionally
correct but the documentation was incomplete.
This patch adds this missing documentation which includes an update to
the documentation for list_for_each_entry_continue_rcu() to match the
documentation for the new list_for_each_entry_from_rcu(), and adds
list_for_each_entry_from_rcu() and the already existing
hlist_for_each_entry_from_rcu() to section 7 of whatisRCU.txt.
Reviewed-by: Paul E. McKenney <[email protected]>
Signed-off-by: NeilBrown <[email protected]>
---
Documentation/RCU/whatisRCU.txt | 2 ++
include/linux/rculist.h | 19 ++++++++++++++++++-
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt
index 65eb856526b7..d43524493fb3 100644
--- a/Documentation/RCU/whatisRCU.txt
+++ b/Documentation/RCU/whatisRCU.txt
@@ -816,11 +816,13 @@ RCU list traversal:
list_next_rcu
list_for_each_entry_rcu
list_for_each_entry_continue_rcu
+ list_for_each_entry_from_rcu
hlist_first_rcu
hlist_next_rcu
hlist_pprev_rcu
hlist_for_each_entry_rcu
hlist_for_each_entry_rcu_bh
+ hlist_for_each_entry_from_rcu
hlist_for_each_entry_continue_rcu
hlist_for_each_entry_continue_rcu_bh
hlist_nulls_first_rcu
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 36df6ccbc874..4786c2235b98 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -396,7 +396,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
* @member: the name of the list_head within the struct.
*
* Continue to iterate over list of given type, continuing after
- * the current position.
+ * the current position which must have been in the list when the RCU read
+ * lock was taken.
+ * This would typically require either that you obtained the node from a
+ * previous walk of the list in the same RCU read-side critical section, or
+ * that you held some sort of non-RCU reference (such as a reference count)
+ * to keep the node alive *and* in the list.
+ *
+ * This iterator is similar to list_for_each_entry_from_rcu() except
+ * this starts after the given position and that one starts at the given
+ * position.
*/
#define list_for_each_entry_continue_rcu(pos, head, member) \
for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
@@ -411,6 +420,14 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
*
* Iterate over the tail of a list starting from a given position,
* which must have been in the list when the RCU read lock was taken.
+ * This would typically require either that you obtained the node from a
+ * previous walk of the list in the same RCU read-side critical section, or
+ * that you held some sort of non-RCU reference (such as a reference count)
+ * to keep the node alive *and* in the list.
+ *
+ * This iterator is similar to list_for_each_entry_continue_rcu() except
+ * this starts from the given position and that one starts from the position
+ * after the given position.
*/
#define list_for_each_entry_from_rcu(pos, head, member) \
for (; &(pos)->member != (head); \
--
2.14.0.rc0.dirty
On Mon, Jun 18, 2018 at 02:22:40PM +1000, NeilBrown wrote:
>
> Unfortunately the patch for adding list_for_each_entry_from_rcu()
> wasn't the final patch after all review. It is functionally
> correct but the documentation was incomplete.
>
> This patch adds this missing documentation which includes an update to
> the documentation for list_for_each_entry_continue_rcu() to match the
> documentation for the new list_for_each_entry_from_rcu(), and adds
> list_for_each_entry_from_rcu() and the already existing
> hlist_for_each_entry_from_rcu() to section 7 of whatisRCU.txt.
>
> Reviewed-by: Paul E. McKenney <[email protected]>
> Signed-off-by: NeilBrown <[email protected]>
Once I rebase my commits to v4.18-rc1, I will apply this, thank you!
(If you were wanting something else to happen with this patch, please
let me know.)
Thanx, Paul
> ---
> Documentation/RCU/whatisRCU.txt | 2 ++
> include/linux/rculist.h | 19 ++++++++++++++++++-
> 2 files changed, 20 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt
> index 65eb856526b7..d43524493fb3 100644
> --- a/Documentation/RCU/whatisRCU.txt
> +++ b/Documentation/RCU/whatisRCU.txt
> @@ -816,11 +816,13 @@ RCU list traversal:
> list_next_rcu
> list_for_each_entry_rcu
> list_for_each_entry_continue_rcu
> + list_for_each_entry_from_rcu
> hlist_first_rcu
> hlist_next_rcu
> hlist_pprev_rcu
> hlist_for_each_entry_rcu
> hlist_for_each_entry_rcu_bh
> + hlist_for_each_entry_from_rcu
> hlist_for_each_entry_continue_rcu
> hlist_for_each_entry_continue_rcu_bh
> hlist_nulls_first_rcu
> diff --git a/include/linux/rculist.h b/include/linux/rculist.h
> index 36df6ccbc874..4786c2235b98 100644
> --- a/include/linux/rculist.h
> +++ b/include/linux/rculist.h
> @@ -396,7 +396,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
> * @member: the name of the list_head within the struct.
> *
> * Continue to iterate over list of given type, continuing after
> - * the current position.
> + * the current position which must have been in the list when the RCU read
> + * lock was taken.
> + * This would typically require either that you obtained the node from a
> + * previous walk of the list in the same RCU read-side critical section, or
> + * that you held some sort of non-RCU reference (such as a reference count)
> + * to keep the node alive *and* in the list.
> + *
> + * This iterator is similar to list_for_each_entry_from_rcu() except
> + * this starts after the given position and that one starts at the given
> + * position.
> */
> #define list_for_each_entry_continue_rcu(pos, head, member) \
> for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
> @@ -411,6 +420,14 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
> *
> * Iterate over the tail of a list starting from a given position,
> * which must have been in the list when the RCU read lock was taken.
> + * This would typically require either that you obtained the node from a
> + * previous walk of the list in the same RCU read-side critical section, or
> + * that you held some sort of non-RCU reference (such as a reference count)
> + * to keep the node alive *and* in the list.
> + *
> + * This iterator is similar to list_for_each_entry_continue_rcu() except
> + * this starts from the given position and that one starts from the position
> + * after the given position.
> */
> #define list_for_each_entry_from_rcu(pos, head, member) \
> for (; &(pos)->member != (head); \
> --
> 2.14.0.rc0.dirty
>
On Mon, Jun 18, 2018 at 10:01:11AM -0700, Paul E. McKenney wrote:
> On Mon, Jun 18, 2018 at 02:22:40PM +1000, NeilBrown wrote:
> >
> > Unfortunately the patch for adding list_for_each_entry_from_rcu()
> > wasn't the final patch after all review. It is functionally
> > correct but the documentation was incomplete.
> >
> > This patch adds this missing documentation which includes an update to
> > the documentation for list_for_each_entry_continue_rcu() to match the
> > documentation for the new list_for_each_entry_from_rcu(), and adds
> > list_for_each_entry_from_rcu() and the already existing
> > hlist_for_each_entry_from_rcu() to section 7 of whatisRCU.txt.
> >
> > Reviewed-by: Paul E. McKenney <[email protected]>
> > Signed-off-by: NeilBrown <[email protected]>
>
> Once I rebase my commits to v4.18-rc1, I will apply this, thank you!
>
> (If you were wanting something else to happen with this patch, please
> let me know.)
And hearing no objections, I have applied it.
Thanx, Paul