Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
disconnected data server) we've been sending layoutreturn calls
while there is potentially still outstanding I/O to the data
servers. The reason we do this is to avoid races between replayed
writes to the MDS and the original writes to the DS.
When this happens, the BUG_ON() in nfs4_layoutreturn_done can
be triggered because it assumes that we would never call
layoutreturn without knowing that all I/O to the DS is
finished. The fix is to remove the BUG_ON() now that the
assumptions behind the test are obsolete.
Reported-by: Boaz Harrosh <[email protected]>
Reported-by: Tigran Mkrtchyan <[email protected]>
Signed-off-by: Trond Myklebust <[email protected]>
Cc: [email protected] [>=3.5]
---
fs/nfs/nfs4proc.c | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index f94f6b3..c77d296 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -6359,12 +6359,8 @@ static void nfs4_layoutreturn_done(struct rpc_task *task, void *calldata)
return;
}
spin_lock(&lo->plh_inode->i_lock);
- if (task->tk_status == 0) {
- if (lrp->res.lrs_present) {
- pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
- } else
- BUG_ON(!list_empty(&lo->plh_segs));
- }
+ if (task->tk_status == 0 && lrp->res.lrs_present)
+ pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
lo->plh_block_lgets--;
spin_unlock(&lo->plh_inode->i_lock);
dprintk("<-- %s\n", __func__);
--
1.7.11.2
T24gVHVlLCAyMDEyLTA4LTE0IGF0IDEwOjQ4ICswMzAwLCBCZW5ueSBIYWxldnkgd3JvdGU6DQo+
IE9uIDIwMTItMDgtMDkgMTg6MzksIE15a2xlYnVzdCwgVHJvbmQgd3JvdGU6DQo+ID4gT24gVGh1
LCAyMDEyLTA4LTA5IGF0IDIzOjAxICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gPj4gT24gVGh1
LCBBdWcgOSwgMjAxMiBhdCAxMDozNiBQTSwgTXlrbGVidXN0LCBUcm9uZA0KPiA+PiA8VHJvbmQu
TXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3RlOg0KPiA+Pj4gT24gVGh1LCAyMDEyLTA4LTA5IGF0
IDIyOjMwICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gPj4+PiBPbiBUaHUsIEF1ZyA5LCAyMDEy
IGF0IDQ6MjEgQU0sIFRyb25kIE15a2xlYnVzdA0KPiA+Pj4+IDxUcm9uZC5NeWtsZWJ1c3RAbmV0
YXBwLmNvbT4gd3JvdGU6DQo+ID4+Pj4+IEV2ZXIgc2luY2UgY29tbWl0IDBhNTdjZGFjM2YgKE5G
U3Y0LjEgc2VuZCBsYXlvdXRyZXR1cm4gdG8gZmVuY2UNCj4gPj4+Pj4gZGlzY29ubmVjdGVkIGRh
dGEgc2VydmVyKSB3ZSd2ZSBiZWVuIHNlbmRpbmcgbGF5b3V0cmV0dXJuIGNhbGxzDQo+ID4+Pj4+
IHdoaWxlIHRoZXJlIGlzIHBvdGVudGlhbGx5IHN0aWxsIG91dHN0YW5kaW5nIEkvTyB0byB0aGUg
ZGF0YQ0KPiA+Pj4+PiBzZXJ2ZXJzLiBUaGUgcmVhc29uIHdlIGRvIHRoaXMgaXMgdG8gYXZvaWQg
cmFjZXMgYmV0d2VlbiByZXBsYXllZA0KPiA+Pj4+PiB3cml0ZXMgdG8gdGhlIE1EUyBhbmQgdGhl
IG9yaWdpbmFsIHdyaXRlcyB0byB0aGUgRFMuDQo+ID4+Pj4+DQo+ID4+Pj4+IFdoZW4gdGhpcyBo
YXBwZW5zLCB0aGUgQlVHX09OKCkgaW4gbmZzNF9sYXlvdXRyZXR1cm5fZG9uZSBjYW4NCj4gPj4+
Pj4gYmUgdHJpZ2dlcmVkIGJlY2F1c2UgaXQgYXNzdW1lcyB0aGF0IHdlIHdvdWxkIG5ldmVyIGNh
bGwNCj4gPj4+Pj4gbGF5b3V0cmV0dXJuIHdpdGhvdXQga25vd2luZyB0aGF0IGFsbCBJL08gdG8g
dGhlIERTIGlzDQo+ID4+Pj4+IGZpbmlzaGVkLiBUaGUgZml4IGlzIHRvIHJlbW92ZSB0aGUgQlVH
X09OKCkgbm93IHRoYXQgdGhlDQo+ID4+Pj4+IGFzc3VtcHRpb25zIGJlaGluZCB0aGUgdGVzdCBh
cmUgb2Jzb2xldGUuDQo+ID4+Pj4+DQo+ID4+Pj4gSXNuJ3QgTURTIHN1cHBvc2VkIHRvIHJlY2Fs
bCB0aGUgbGF5b3V0IGlmIHJhY2VzIGFyZSBwb3NzaWJsZSBiZXR3ZWVuDQo+ID4+Pj4gb3V0c3Rh
bmRpbmcgd3JpdGUtdG8tRFMgYW5kIHdyaXRlLXRocm91Z2gtTURTPw0KPiA+Pj4NCj4gPj4+IFdo
ZXJlIGRvIHlvdSByZWFkIHRoYXQgaW4gUkZDNTY2MT8NCj4gPj4+DQo+ID4+IFRoYXQncyBteSAo
bWF5YmUgbWlzLSl1bmRlcnN0YW5kaW5nIG9mIGhvdyBzZXJ2ZXIgd29ya3MuLi4gQnV0IGxvb2tp
bmcNCj4gPj4gYXQgcmZjNTY2MSBzZWN0aW9uIDE4LjQ0LjMuIGxheW91dHJldHVybiBpbXBsZW1l
bnRhdGlvbi4NCj4gPj4gIg0KPiA+PiBBZnRlciB0aGlzIGNhbGwsDQo+ID4+ICAgIHRoZSBjbGll
bnQgTVVTVCBOT1QgdXNlIHRoZSByZXR1cm5lZCBsYXlvdXQocykgYW5kIHRoZSBhc3NvY2lhdGVk
DQo+ID4+ICAgIHN0b3JhZ2UgcHJvdG9jb2wgdG8gYWNjZXNzIHRoZSBmaWxlIGRhdGEuDQo+ID4+
ICINCj4gPj4gQW5kIGdpdmVuIGNvbW1pdCAwYTU3Y2RhYzNmLCBjbGllbnQgaXMgdXNpbmcgdGhl
IGxheW91dCBldmVuIGFmdGVyDQo+ID4+IGxheW91dHJldHVybiwgd2hpY2ggSU1ITyBpcyBhIHZp
b2xhdGlvbiBvZiByZmM1NjYxLg0KPiA+IA0KPiA+IE5vLiBJdCBpcyB1c2luZyB0aGUgbGF5b3V0
cmV0dXJuIHRvIHRlbGwgdGhlIE1EUyB0byBmZW5jZSBvZmYgSS9PIHRvIGENCj4gPiBkYXRhIHNl
cnZlciB0aGF0IGlzIG5vdCByZXNwb25kaW5nLiBJdCBpc24ndCBhdHRlbXB0aW5nIHRvIHVzZSB0
aGUNCj4gPiBsYXlvdXQgYWZ0ZXIgdGhlIGxheW91dHJldHVybjogdGhlIHdob2xlIHBvaW50IGlz
IHRoYXQgd2UgYXJlIGF0dGVtcHRpbmcNCj4gPiB3cml0ZS10aHJvdWdoLU1EUyBhZnRlciB0aGUg
YXR0ZW1wdCB0byB3cml0ZSB0aHJvdWdoIHRoZSBEUyB0aW1lZCBvdXQuDQo+ID4gDQo+IA0KPiBJ
IGhlYXIgeW91LCBidXQgdGhpcyB1c2UgY2FzZSBpcyB2YWxpZCBhZnRlciBhIHRpbWUgb3V0IC8g
ZGlzY29ubmVjdA0KPiAod2hpY2ggd2lsbCB0cmFuc2xhdGUgdG8gUE5GU19PU0RfRVJSX1VOUkVB
Q0hBQkxFIGZvciB0aGUgb2JqZWN0cyBsYXlvdXQpDQo+IEluIG90aGVyIGNhc2VzLCBJL09zIHRv
IHRoZSBEUyBtaWdodCBvYnZpb3VzbHkgYmUgaW4gZmxpZ2h0IGFuZCB0aGUgQlVHX09ODQo+IGlu
ZGljYXRlcyB0aGF0Lg0KPiANCj4gSU1PLCB0aGUgcmlnaHQgd2F5IHRvIGltcGxlbWVudCB0aGF0
IGlzIHRvIGluaXRpYWxseSBtYXJrIHRoZSBsc2VncyBpbnZhbGlkDQo+IGFuZCBpbmNyZW1lbnQg
cGxoX2Jsb2NrX2xnZXRzLCBhcyB3ZSBkbyB0b2RheSBpbiBfcG5mc19yZXR1cm5fbGF5b3V0DQo+
IGJ1dCBhY3R1YWxseSBzZW5kIHRoZSBsYXlvdXQgcmV0dXJuIG9ubHkgd2hlbiB0aGUgbGFzdCBz
ZWdtZW50IGlzIGRlcmVmZXJlbmNlZC4NCg0KVGhpcyBpcyB3aGF0IHdlIGRvIGZvciBvYmplY3Qg
YW5kIGJsb2NrIGxheW91dCB0eXBlcywgc28geW91cg0Kb2JqZWN0cy1zcGVjaWZpYyBvYmplY3Rp
b24gaXMgdW5mb3VuZGVkLg0KDQpBcyBJIHVuZGVyc3RhbmQgaXQsIGlTQ1NJIGhhcyBkaWZmZXJl
bnQgc2VtYW50aWNzIHcuci50LiBkaXNjb25uZWN0IGFuZA0KdGltZW91dCwgd2hpY2ggbWVhbnMg
dGhhdCB0aGUgY2xpZW50IGNhbiBpbiBwcmluY2lwbGUgcmVseSBvbiBhIHRpbWVvdXQNCmxlYXZp
bmcgdGhlIERTIGluIGEga25vd24gc3RhdGUuIERpdHRvIGZvciBGQ1AuDQpJJ3ZlIG5vIGlkZWEg
YWJvdXQgb3RoZXIgYmxvY2svb2JqZWN0IHRyYW5zcG9ydCB0eXBlcywgYnV0IEkgYXNzdW1lDQp0
aG9zZSB0aGF0IHN1cHBvcnQgbXVsdGktcGF0aGluZyBpbXBsZW1lbnQgc2ltaWxhciBkZXZpY2Vz
Lg0KDQpUaGUgcHJvYmxlbSBpcyB0aGF0IFJQQyBkb2VzIG5vdCwgc28gdGhlIGZpbGVzIGxheW91
dCBuZWVkcyB0byBiZQ0KdHJlYXRlZCBkaWZmZXJlbnRseS4NCg0KLS0gDQpUcm9uZCBNeWtsZWJ1
c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0KDQpOZXRBcHANClRyb25kLk15a2xlYnVz
dEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=
T24gVGh1LCAyMDEyLTA4LTA5IGF0IDIzOjAxICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gT24g
VGh1LCBBdWcgOSwgMjAxMiBhdCAxMDozNiBQTSwgTXlrbGVidXN0LCBUcm9uZA0KPiA8VHJvbmQu
TXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3RlOg0KPiA+IE9uIFRodSwgMjAxMi0wOC0wOSBhdCAy
MjozMCArMDgwMCwgUGVuZyBUYW8gd3JvdGU6DQo+ID4+IE9uIFRodSwgQXVnIDksIDIwMTIgYXQg
NDoyMSBBTSwgVHJvbmQgTXlrbGVidXN0DQo+ID4+IDxUcm9uZC5NeWtsZWJ1c3RAbmV0YXBwLmNv
bT4gd3JvdGU6DQo+ID4+ID4gRXZlciBzaW5jZSBjb21taXQgMGE1N2NkYWMzZiAoTkZTdjQuMSBz
ZW5kIGxheW91dHJldHVybiB0byBmZW5jZQ0KPiA+PiA+IGRpc2Nvbm5lY3RlZCBkYXRhIHNlcnZl
cikgd2UndmUgYmVlbiBzZW5kaW5nIGxheW91dHJldHVybiBjYWxscw0KPiA+PiA+IHdoaWxlIHRo
ZXJlIGlzIHBvdGVudGlhbGx5IHN0aWxsIG91dHN0YW5kaW5nIEkvTyB0byB0aGUgZGF0YQ0KPiA+
PiA+IHNlcnZlcnMuIFRoZSByZWFzb24gd2UgZG8gdGhpcyBpcyB0byBhdm9pZCByYWNlcyBiZXR3
ZWVuIHJlcGxheWVkDQo+ID4+ID4gd3JpdGVzIHRvIHRoZSBNRFMgYW5kIHRoZSBvcmlnaW5hbCB3
cml0ZXMgdG8gdGhlIERTLg0KPiA+PiA+DQo+ID4+ID4gV2hlbiB0aGlzIGhhcHBlbnMsIHRoZSBC
VUdfT04oKSBpbiBuZnM0X2xheW91dHJldHVybl9kb25lIGNhbg0KPiA+PiA+IGJlIHRyaWdnZXJl
ZCBiZWNhdXNlIGl0IGFzc3VtZXMgdGhhdCB3ZSB3b3VsZCBuZXZlciBjYWxsDQo+ID4+ID4gbGF5
b3V0cmV0dXJuIHdpdGhvdXQga25vd2luZyB0aGF0IGFsbCBJL08gdG8gdGhlIERTIGlzDQo+ID4+
ID4gZmluaXNoZWQuIFRoZSBmaXggaXMgdG8gcmVtb3ZlIHRoZSBCVUdfT04oKSBub3cgdGhhdCB0
aGUNCj4gPj4gPiBhc3N1bXB0aW9ucyBiZWhpbmQgdGhlIHRlc3QgYXJlIG9ic29sZXRlLg0KPiA+
PiA+DQo+ID4+IElzbid0IE1EUyBzdXBwb3NlZCB0byByZWNhbGwgdGhlIGxheW91dCBpZiByYWNl
cyBhcmUgcG9zc2libGUgYmV0d2Vlbg0KPiA+PiBvdXRzdGFuZGluZyB3cml0ZS10by1EUyBhbmQg
d3JpdGUtdGhyb3VnaC1NRFM/DQo+ID4NCj4gPiBXaGVyZSBkbyB5b3UgcmVhZCB0aGF0IGluIFJG
QzU2NjE/DQo+ID4NCj4gVGhhdCdzIG15IChtYXliZSBtaXMtKXVuZGVyc3RhbmRpbmcgb2YgaG93
IHNlcnZlciB3b3Jrcy4uLiBCdXQgbG9va2luZw0KPiBhdCByZmM1NjYxIHNlY3Rpb24gMTguNDQu
My4gbGF5b3V0cmV0dXJuIGltcGxlbWVudGF0aW9uLg0KPiAiDQo+IEFmdGVyIHRoaXMgY2FsbCwN
Cj4gICAgdGhlIGNsaWVudCBNVVNUIE5PVCB1c2UgdGhlIHJldHVybmVkIGxheW91dChzKSBhbmQg
dGhlIGFzc29jaWF0ZWQNCj4gICAgc3RvcmFnZSBwcm90b2NvbCB0byBhY2Nlc3MgdGhlIGZpbGUg
ZGF0YS4NCj4gIg0KPiBBbmQgZ2l2ZW4gY29tbWl0IDBhNTdjZGFjM2YsIGNsaWVudCBpcyB1c2lu
ZyB0aGUgbGF5b3V0IGV2ZW4gYWZ0ZXINCj4gbGF5b3V0cmV0dXJuLCB3aGljaCBJTUhPIGlzIGEg
dmlvbGF0aW9uIG9mIHJmYzU2NjEuDQoNCk5vLiBJdCBpcyB1c2luZyB0aGUgbGF5b3V0cmV0dXJu
IHRvIHRlbGwgdGhlIE1EUyB0byBmZW5jZSBvZmYgSS9PIHRvIGENCmRhdGEgc2VydmVyIHRoYXQg
aXMgbm90IHJlc3BvbmRpbmcuIEl0IGlzbid0IGF0dGVtcHRpbmcgdG8gdXNlIHRoZQ0KbGF5b3V0
IGFmdGVyIHRoZSBsYXlvdXRyZXR1cm46IHRoZSB3aG9sZSBwb2ludCBpcyB0aGF0IHdlIGFyZSBh
dHRlbXB0aW5nDQp3cml0ZS10aHJvdWdoLU1EUyBhZnRlciB0aGUgYXR0ZW1wdCB0byB3cml0ZSB0
aHJvdWdoIHRoZSBEUyB0aW1lZCBvdXQuDQoNCj4gPj4gQW5kIGl0IGNhdXNlcyBkYXRhIGNvcnJ1
cHRpb24gZm9yIGJsb2NrbGF5b3V0IGlmIGNsaWVudCByZXR1cm5zIGxheW91dA0KPiA+PiB3aGls
ZSB0aGVyZSBpcyBpbi1mbGlnaHQgZGlzayBJTy4uLg0KPiA+DQo+ID4gVGhlbiBpdCBuZWVkcyB0
byB0dXJuIG9mZiBmYXN0IGZhaWxvdmVyIHRvIHdyaXRlLXRocm91Z2gtTURTLg0KPiA+DQo+IElm
IHlvdSBzdGlsbCBjb25zaWRlciBpdCBmb2xsb3dpbmcgcmZjNTY2MSwgSSdkIGNob29zZSB0byBk
aXNhYmxlDQo+IGxheW91dHJldHVybiBpbiBiZWZvcmUgd3JpdGUtdGhyb3VnaC1NRFMgZm9yIGJs
b2NrbGF5b3V0LCBieSBhZGRpbmcNCj4gc29tZSBmbGFnIGxpa2UgUE5GU19OT19MQVlPVVRSRVRf
T05fRkFMTFRIUlUgc2ltaWxhciB0byBvYmplY3RzJw0KPiBQTkZTX0xBWU9VVFJFVF9PTl9TRVRB
VFRSLg0KDQpJIGRvbid0IHNlZSBob3cgdGhhdCB3aWxsIHByZXZlbnQgY29ycnVwdGlvbi4NCg0K
SW4gZmFjdCwgSUlSQywgaW4gTkZTdjQuMiBTb3JpbidzIHByb3Bvc2VkIGNoYW5nZXMgc3BlY2lm
aWNhbGx5IHVzZSB0aGUNCmxheW91dHJldHVybiB0byBjb21tdW5pY2F0ZSB0byB0aGUgTURTIHRo
YXQgdGhlIERTIGlzIHRpbWluZyBvdXQgdmlhIGFuDQplcnJvciBjb2RlIChsaWtlIHRoZSBvYmpl
Y3QgbGF5b3V0IGhhcyBkb25lIGFsbCB0aGUgdGltZSkuIEhvdyBjYW4geW91DQpyZWNvbmNpbGUg
dGhhdCBjaGFuZ2Ugd2l0aCBhIGZsYWcgc3VjaCBhcyB0aGUgb25lIHlvdSBwcm9wb3NlPw0KDQot
LSANClRyb25kIE15a2xlYnVzdA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFw
cA0KVHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQoNCg==
On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
<[email protected]> wrote:
> Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
> disconnected data server) we've been sending layoutreturn calls
> while there is potentially still outstanding I/O to the data
> servers. The reason we do this is to avoid races between replayed
> writes to the MDS and the original writes to the DS.
>
> When this happens, the BUG_ON() in nfs4_layoutreturn_done can
> be triggered because it assumes that we would never call
> layoutreturn without knowing that all I/O to the DS is
> finished. The fix is to remove the BUG_ON() now that the
> assumptions behind the test are obsolete.
>
Isn't MDS supposed to recall the layout if races are possible between
outstanding write-to-DS and write-through-MDS?
And it causes data corruption for blocklayout if client returns layout
while there is in-flight disk IO...
--
Thanks,
Tao
On 2012-08-14 17:53, Myklebust, Trond wrote:
> On Tue, 2012-08-14 at 22:30 +0800, Peng Tao wrote:
>> On Tue, Aug 14, 2012 at 9:45 PM, Myklebust, Trond
>> <[email protected]> wrote:
>>> On Tue, 2012-08-14 at 10:48 +0300, Benny Halevy wrote:
>>>> On 2012-08-09 18:39, Myklebust, Trond wrote:
>>>>> On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>>>>>> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>>>>>> <[email protected]> wrote:
>>>>>>> On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>>>>>>>> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>>>>>>>> <[email protected]> wrote:
>>>>>>>>> Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>>>>>>>>> disconnected data server) we've been sending layoutreturn calls
>>>>>>>>> while there is potentially still outstanding I/O to the data
>>>>>>>>> servers. The reason we do this is to avoid races between replayed
>>>>>>>>> writes to the MDS and the original writes to the DS.
>>>>>>>>>
>>>>>>>>> When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>>>>>>>>> be triggered because it assumes that we would never call
>>>>>>>>> layoutreturn without knowing that all I/O to the DS is
>>>>>>>>> finished. The fix is to remove the BUG_ON() now that the
>>>>>>>>> assumptions behind the test are obsolete.
>>>>>>>>>
>>>>>>>> Isn't MDS supposed to recall the layout if races are possible between
>>>>>>>> outstanding write-to-DS and write-through-MDS?
>>>>>>>
>>>>>>> Where do you read that in RFC5661?
>>>>>>>
>>>>>> That's my (maybe mis-)understanding of how server works... But looking
>>>>>> at rfc5661 section 18.44.3. layoutreturn implementation.
>>>>>> "
>>>>>> After this call,
>>>>>> the client MUST NOT use the returned layout(s) and the associated
>>>>>> storage protocol to access the file data.
>>>>>> "
>>>>>> And given commit 0a57cdac3f, client is using the layout even after
>>>>>> layoutreturn, which IMHO is a violation of rfc5661.
>>>>>
>>>>> No. It is using the layoutreturn to tell the MDS to fence off I/O to a
>>>>> data server that is not responding. It isn't attempting to use the
>>>>> layout after the layoutreturn: the whole point is that we are attempting
>>>>> write-through-MDS after the attempt to write through the DS timed out.
>>>>>
>>>>
>>>> I hear you, but this use case is valid after a time out / disconnect
>>>> (which will translate to PNFS_OSD_ERR_UNREACHABLE for the objects layout)
>>>> In other cases, I/Os to the DS might obviously be in flight and the BUG_ON
>>>> indicates that.
>>>>
>>>> IMO, the right way to implement that is to initially mark the lsegs invalid
>>>> and increment plh_block_lgets, as we do today in _pnfs_return_layout
>>>> but actually send the layout return only when the last segment is dereferenced.
>>>
>>> This is what we do for object and block layout types, so your
>>> objects-specific objection is unfounded.
>>>
>> object layout is also doing layout return on IO error (commit
>> fe0fe83585f8). And it doesn't take care of draining concurrent
>> in-flight IO. I guess that's why Boaz saw the same BUG_ON.
>
> Yes. I did notice that code when I was looking into this. However that's
> Boaz's own patch, and it _only_ applies to the objects layout type. I
> assumed that he had tested it when I applied it...
>
> One way to fix that would be to keep a count of "outstanding
> read/writes" in the layout, so that when the error occurs, and we want
> to fall back to MDS, we just increment plh_block_lgets, invalidate the
> layout, and then let the outstanding read/writes fall to zero before
> sending the layoutreturn.
Sounds reasonable to me too.
> If the objects layout wants to do that, then I have no objection. As
> I've said multiple times, though, I'm not convinced we want to do that
> for the files layout.
>
I just fear that removing the BUG_ON will prevent us from detecting cases
where a LAYOUTRETURN is sent while there are layout segments in use
in the error free or non-timeout case.
Benny
On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
<[email protected]> wrote:
> On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>> <[email protected]> wrote:
>> > Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>> > disconnected data server) we've been sending layoutreturn calls
>> > while there is potentially still outstanding I/O to the data
>> > servers. The reason we do this is to avoid races between replayed
>> > writes to the MDS and the original writes to the DS.
>> >
>> > When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>> > be triggered because it assumes that we would never call
>> > layoutreturn without knowing that all I/O to the DS is
>> > finished. The fix is to remove the BUG_ON() now that the
>> > assumptions behind the test are obsolete.
>> >
>> Isn't MDS supposed to recall the layout if races are possible between
>> outstanding write-to-DS and write-through-MDS?
>
> Where do you read that in RFC5661?
>
That's my (maybe mis-)understanding of how server works... But looking
at rfc5661 section 18.44.3. layoutreturn implementation.
"
After this call,
the client MUST NOT use the returned layout(s) and the associated
storage protocol to access the file data.
"
And given commit 0a57cdac3f, client is using the layout even after
layoutreturn, which IMHO is a violation of rfc5661.
>> And it causes data corruption for blocklayout if client returns layout
>> while there is in-flight disk IO...
>
> Then it needs to turn off fast failover to write-through-MDS.
>
If you still consider it following rfc5661, I'd choose to disable
layoutreturn in before write-through-MDS for blocklayout, by adding
some flag like PNFS_NO_LAYOUTRET_ON_FALLTHRU similar to objects'
PNFS_LAYOUTRET_ON_SETATTR.
--
Thanks,
Tao
On 08/13/2012 07:26 PM, Myklebust, Trond wrote:
>> This above here is where you are wrong!! You don't understand my point,
>> and ignore my comments. So let me state it as clear as I can.
>
> YOU are ignoring the reality of SunRPC. There is no abort/cancel/timeout
> for an RPC call once it has started. This is why we need fencing
> _specifically_ for the pNFS files client.
>
Again we have a communication problem between us. I say some words and
mean one thing, and you say and hear the same words but attach different
meanings to them. This is no one's fault it's just is.
Lets do an experiment, mount a regular NFS4 in -o soft mode and start
writing to server, say with dd. Now disconnect the cable. After some timeout
the dd will return with "IO error", and will stop writing to file.
This is the timeout I mean. Surely some RPC-requests did not complete and
returned to NFS core with some kind of error.
With RPC-requests I do not mean the RPC protocol on the wire, I mean that
entity inside the Linux Kernel which represents an RPC. Surly some
linux-RPC-requests objects were not released do to a server "rpc-done"
received. But do to an internal mechanism that called the "release" method do to
a communication timeout.
So this is what I call "returned with a timeout". It does exist and used
every day.
Even better if I don't disconnect the wire but do an if_down or halt on the
server, the dd's IO error will happen immediately, not even wait for
any timeout. This is because the socket is orderly closed and all
sends/receives will return quickly with a "disconnect-error".
When I use a single Server like the nfs4 above. Then there is one fact
in above scenario that I want to point out:
At some point in the NFS-Core state. There is a point that no more
requests are issued, all old request have released, and an error is
returned to the application. At that point the client will not call
skb-send, and will not try farther communication with the Server.
This is what must happen with ALL DSs that belong to a layout, before
client should be LAYOUT_RETURN(ing). The client can only do it's job. That is:
STOP any skb-send, to any of the DSs in a layout.
Only then it is complying to the RFC.
So this is what I mean by "return with a timeout below"
>> (Lets assume files layout, for blocks and objects it's a bit different
>> but mostly the same.)
>
> That, and the fact that fencing hasn't been implemented for blocks and
> objects.
That's not true. At Panasas and both at EMC there is fencing in place and
it is used every day. This is why I insist that it is very much
the same for all of us.
> The commit in question is 82c7c7a5a (NFSv4.1 return the LAYOUT
> for each file with failed DS connection I/O) and touches only
> fs/nfs/nfs4filelayout.c. It cannot be affecting blocks and objects.
>
OK I had in mind the patches that Andy sent. I'll look again for what
actually went in. (It was all while I was unavailable)
>> - Heavy IO is going on, the device_id in question has *3* DSs in it's
>> device topography. Say DS1, DS2, DS3
>>
>> - We have been queuing IO, and all queues are full. (we have 3 queues in
>> in question, right? What is the maximum Q depth per files-DS? I know
>> that in blocks and objects we usually have, I think, something like 128.
>> This is a *tunable* in the block-layer's request-queue. Is it not some
>> negotiated parameter with the NFS servers?)
>>
>> - Now, boom DS2 has timed-out. The Linux-client resets the session and
>> internally closes all sockets of that session. All the RPCs that
>> belong to DS2 are being returned up with a timeout error. This one
>> is just the first of all those belonging to this DS2. They will
>> be decrementing the reference for this layout very, very soon.
>>
>> - But what about DS1, and DS3 RPCs. What should we do with those?
>> This is where you guys (Trond and Andy) are wrong. We must also
>> wait for these RPC's as well. And opposite to what you think, this
>> should not take long. Let me explain:
>>
>> We don't know anything about DS1 and DS3, each might be, either,
>> "Having the same communication problem, like DS2". Or "is just working
>> fine". So lets say for example that DS3 will also time-out in the
>> future, and that DS1 is just fine and is writing as usual.
>>
>> * DS1 - Since it's working, it has most probably already done
>> with all it's IO, because the NFS timeout is usually much longer
>> then the normal RPC time, and since we are queuing evenly on
>> all 3 DSs, at this point must probably, all of DS1 RPCs are
>> already done. (And layout has been de-referenced).
>>
>> * DS3 - Will timeout in the future, when will that be?
>> So let me start with, saying:
>> (1). We could enhance our code and proactively,
>> "cancel/abort" all RPCs that belong to DS3 (more on this
>> below)
>
> Which makes the race _WORSE_. As I said above, there is no 'cancel RPC'
> operation in SUNRPC. Once your RPC call is launched, it cannot be
> recalled. All your discussion above is about the client side, and
> ignores what may be happening on the data server side. The fencing is
> what is needed to deal with the data server picture.
>
Again, some miss understanding. I never said we should not send
a LAYOUT_RETURN before writing through MDS. The opposite is true,
I think it is a novel idea and gives you the kind of barrier that
will harden and robust the system.
WHAT I'm saying is that this cannot happen while the schizophrenic
client is busily still actively skb-sending more and more bytes
to all the other DSs in the layout. LONG AFTER THE LAYOUT_RETURN
HAS BEEN SENT AND RESPONDED.
So what you are saying does not at all contradicts what I want.
"The fencing is what is needed to deal with the data server picture"
Fine But ONLY after the client has really stopped all sends.
(Each one will do it's job)
BTW: The Server does not *need* the Client to send a LAYOUT_RETURN
It's just a nice-to-have, which I'm fine with.
Both Panasas and EMC when IO is sent through MDS, will first
recall, overlapping layouts, and only then proceed with
MDS processing. (This is some deeply rooted mechanism inside
the FS, an MDS being just another client)
So this is a known problem that is taken care of. But I totally
agree with you, the client LAYOUT_RETURN(ing) the layout will save
lots of protocol time by avoiding the recalls.
Now you understand why in Objects we mandated this LAYOUT_RETURN
on errors. And while at it we want the exact error reported.
>> (2). Or We can prove that DS3's RPCs will timeout at worst
>> case 1 x NFS-timeout after above DS2 timeout event, or
>> 2 x NFS-timeout after the queuing of the first timed-out
>> RPC. And statistically in the average case DS3 will timeout
>> very near the time DS2 timed-out.
>>
>> This is easy since the last IO we queued was the one that
>> made DS2's queue to be full, and it was kept full because
>> DS2 stopped responding and nothing emptied the queue.
>>
>> So the easiest we can do is wait for DS3 to timeout, soon
>> enough, and once that will happen, session will be reset and all
>> RPCs will end with an error.
>
>
> You are still only discussing the client side.
>
> Read my lips: Sun RPC OPERATIONS DO NOT TIMEOUT AND CANNOT BE ABORTED OR
> CANCELED. Fencing is the closest we can come to an abort operation.
>
Again I did not mean the "Sun RPC OPERATIONS" on the wire. I meant
the Linux-request-entity which while exist has a potential to be
submitted for skb-send. As seen above these entities do timeout
in "-o soft" mode and once released remove the potential of any more
future skb-sends on the wire.
BUT what I do not understand is: In above example we are talking
about DS3. We assumed that DS3 has a communication problem. So no amount
of "fencing" or vudu or any other kind of operation can ever affect
the client regarding DS3. Because even if On-the-server pending requests
from client on DS3 are fenced and discarded these errors will not
be communicated back the client. The client will sit idle on DS3
communication until the end of the timeout, regardless.
Actually what I propose for DS3 in the best robust client is to
destroy it's DS3's sessions and therefor cause all Linux-request-entities
to return much much faster, then if *just waiting* for the timeout to expire.
>> So in the worst case scenario we can recover 2 x NFS-timeout after
>> a network partition, which is just 1 x NFS-timeout, after your
>> schizophrenic FENCE_ME_OFF, newly invented operation.
>>
>> What we can do to enhance our code to reduce error recovery to
>> 1 x NFS-timeout:
>>
>> - DS3 above:
>> (As I said DS1's queues are now empty, because it was working fine,
>> So DS3 is a representation of all DS's that have RPCs at the
>> time DS2 timed-out, which belong to this layout)
>>
>> We can proactively abort all RPCs belonging to DS3. If there is
>> a way to internally abort RPC's use that. Else just reset it's
>> session and all sockets will close (and reopen), and all RPC's
>> will end with a disconnect error.
>
> Not on most servers that I'm aware of. If you close or reset the socket
> on the client, then the Linux server will happily continue to process
> those RPC calls; it just won't be able to send a reply.
> Furthermore, if the problem is that the data server isn't responding,
> then a socket close/reset tells you nothing either.
>
Again I'm talking about the NFS-Internal-request-entities these will
be released, though guarantying that no more threads will use any of
these to send any more bytes over to any DSs.
AND yes, yes. Once the client has done it's job and stopped any future
skb-sends to *all* DSs in question, only then it can report to MDS:
"Hey I'm done sending in all other routs here LAYOUT_RETURN"
(Now fencing happens on servers)
and client goes on and says
"Hey can you MDS please also write this data"
Which is perfect for MDS because otherwise if it wants to make sure,
it will need to recall all outstanding layouts, exactly for your
reason, for concern for the data corruption that can happen.
>> - Both DS2 that timed-out, and DS3 that was aborted. Should be
>> marked with a flag. When new IO that belong to some other
>> inode through some other layout+device_id encounters a flagged
>> device, it should abort and turn to MDS IO, with also invalidating
>> it's layout, and hens, soon enough the device_id for DS2&3 will be
>> de-referenced and be removed from device cache. (And all referencing
>> layouts are now gone)
>
> There is no RPC abort functionality Sun RPC. Again, this argument relies
> on functionality that _doesn't_ exist.
>
Again I mean internally at the client. For example closing the socket will
have the effect I want. (And some other tricks we can talk about those
later, lets agree about the principal first)
>> So we do not continue queuing new IO to dead devices. And since most
>> probably MDS will not give us dead servers in new layout, we should be
>> good.
>> In summery.
>> - FENCE_ME_OFF is a new operation, and is not === LAYOUT_RETURN. Client
>> *must not* skb-send a single byte belonging to a layout, after the send
>> of LAYOUT_RETURN.
>> (It need not wait for OPT_DONE from DS to do that, it just must make
>> sure, that all it's internal, or on-the-wire request, are aborted
>> by easily closing the sockets they belong too, and/or waiting for
>> healthy DS's IO to be OPT_DONE . So the client is not dependent on
>> any DS response, it is only dependent on it's internal state being
>> *clean* from any more skb-send(s))
>
> Ditto
>
>> - The proper implementation of LAYOUT_RETURN on error for fast turnover
>> is not hard, and does not involve a new invented NFS operation such
>> as FENCE_ME_OFF. Proper codded client, independently, without
>> the aid of any FENCE_ME_OFF operation, can achieve a faster turnaround
>> by actively returning all layouts that belong to a bad DS, and not
>> waiting for a fence-off of a single layout, then encountering just
>> the same error with all other layouts that have the same DS
>
> What do you mean by "all layouts that belong to a bad DS"? Layouts don't
> belong to a DS, and so there is no way to get from a DS to a layout.
>
Why, sure. loop on all layouts and ask if it has a specific DS.
>> - And I know that just as you did not read my emails from before
>> me going to Hospital, you will continue to not understand this
>> one, or what I'm trying to explain, and will most probably ignore
>> all of it. But please note one thing:
>
> I read them, but just as now, they continue to ignore the reality about
> timeouts: timeouts mean _nothing_ in an RPC failover situation. There is
> no RPC abort functionality that you can rely on other than fencing.
>
I hope I explained this by now. If not please, please, lets organize
a phone call. We can use Panasas conference number whenever you are
available. I think we communicate better in person.
Everyone else is also invited.
BUT there is one most important point for me:
As stated by the RFC. Client must guaranty that no more bytes will be
sent to any DSs in a layout, once LAYOUT_RETURN is sent. This is the
only definition of LAYOUT_RETURN, and NO_MATCHING_LAYOUT as response
to a LAYOUT_RECALL. Which is:
Client has indicated no more future sends on a layout. (And server will
enforce it with a fencing)
>> YOU have sabotaged the NFS 4.1 Linux client, which is now totally
>> not STD complaint, and have introduced CRASHs. And for no good
>> reason.
>
> See above.
>
OK We'll have to see about these crashes, lets talk about them.
Thanks
Boaz
T24gRnJpLCAyMDEyLTA4LTEwIGF0IDAwOjIyICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gT24g
VGh1LCBBdWcgOSwgMjAxMiBhdCAxMTozOSBQTSwgTXlrbGVidXN0LCBUcm9uZA0KPiA8VHJvbmQu
TXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3RlOg0KPiA+IE9uIFRodSwgMjAxMi0wOC0wOSBhdCAy
MzowMSArMDgwMCwgUGVuZyBUYW8gd3JvdGU6DQo+ID4+IE9uIFRodSwgQXVnIDksIDIwMTIgYXQg
MTA6MzYgUE0sIE15a2xlYnVzdCwgVHJvbmQNCj4gPj4gPFRyb25kLk15a2xlYnVzdEBuZXRhcHAu
Y29tPiB3cm90ZToNCj4gPj4gPiBPbiBUaHUsIDIwMTItMDgtMDkgYXQgMjI6MzAgKzA4MDAsIFBl
bmcgVGFvIHdyb3RlOg0KPiA+PiA+PiBPbiBUaHUsIEF1ZyA5LCAyMDEyIGF0IDQ6MjEgQU0sIFRy
b25kIE15a2xlYnVzdA0KPiA+PiA+PiA8VHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3Rl
Og0KPiA+PiA+PiA+IEV2ZXIgc2luY2UgY29tbWl0IDBhNTdjZGFjM2YgKE5GU3Y0LjEgc2VuZCBs
YXlvdXRyZXR1cm4gdG8gZmVuY2UNCj4gPj4gPj4gPiBkaXNjb25uZWN0ZWQgZGF0YSBzZXJ2ZXIp
IHdlJ3ZlIGJlZW4gc2VuZGluZyBsYXlvdXRyZXR1cm4gY2FsbHMNCj4gPj4gPj4gPiB3aGlsZSB0
aGVyZSBpcyBwb3RlbnRpYWxseSBzdGlsbCBvdXRzdGFuZGluZyBJL08gdG8gdGhlIGRhdGENCj4g
Pj4gPj4gPiBzZXJ2ZXJzLiBUaGUgcmVhc29uIHdlIGRvIHRoaXMgaXMgdG8gYXZvaWQgcmFjZXMg
YmV0d2VlbiByZXBsYXllZA0KPiA+PiA+PiA+IHdyaXRlcyB0byB0aGUgTURTIGFuZCB0aGUgb3Jp
Z2luYWwgd3JpdGVzIHRvIHRoZSBEUy4NCj4gPj4gPj4gPg0KPiA+PiA+PiA+IFdoZW4gdGhpcyBo
YXBwZW5zLCB0aGUgQlVHX09OKCkgaW4gbmZzNF9sYXlvdXRyZXR1cm5fZG9uZSBjYW4NCj4gPj4g
Pj4gPiBiZSB0cmlnZ2VyZWQgYmVjYXVzZSBpdCBhc3N1bWVzIHRoYXQgd2Ugd291bGQgbmV2ZXIg
Y2FsbA0KPiA+PiA+PiA+IGxheW91dHJldHVybiB3aXRob3V0IGtub3dpbmcgdGhhdCBhbGwgSS9P
IHRvIHRoZSBEUyBpcw0KPiA+PiA+PiA+IGZpbmlzaGVkLiBUaGUgZml4IGlzIHRvIHJlbW92ZSB0
aGUgQlVHX09OKCkgbm93IHRoYXQgdGhlDQo+ID4+ID4+ID4gYXNzdW1wdGlvbnMgYmVoaW5kIHRo
ZSB0ZXN0IGFyZSBvYnNvbGV0ZS4NCj4gPj4gPj4gPg0KPiA+PiA+PiBJc24ndCBNRFMgc3VwcG9z
ZWQgdG8gcmVjYWxsIHRoZSBsYXlvdXQgaWYgcmFjZXMgYXJlIHBvc3NpYmxlIGJldHdlZW4NCj4g
Pj4gPj4gb3V0c3RhbmRpbmcgd3JpdGUtdG8tRFMgYW5kIHdyaXRlLXRocm91Z2gtTURTPw0KPiA+
PiA+DQo+ID4+ID4gV2hlcmUgZG8geW91IHJlYWQgdGhhdCBpbiBSRkM1NjYxPw0KPiA+PiA+DQo+
ID4+IFRoYXQncyBteSAobWF5YmUgbWlzLSl1bmRlcnN0YW5kaW5nIG9mIGhvdyBzZXJ2ZXIgd29y
a3MuLi4gQnV0IGxvb2tpbmcNCj4gPj4gYXQgcmZjNTY2MSBzZWN0aW9uIDE4LjQ0LjMuIGxheW91
dHJldHVybiBpbXBsZW1lbnRhdGlvbi4NCj4gPj4gIg0KPiA+PiBBZnRlciB0aGlzIGNhbGwsDQo+
ID4+ICAgIHRoZSBjbGllbnQgTVVTVCBOT1QgdXNlIHRoZSByZXR1cm5lZCBsYXlvdXQocykgYW5k
IHRoZSBhc3NvY2lhdGVkDQo+ID4+ICAgIHN0b3JhZ2UgcHJvdG9jb2wgdG8gYWNjZXNzIHRoZSBm
aWxlIGRhdGEuDQo+ID4+ICINCj4gPj4gQW5kIGdpdmVuIGNvbW1pdCAwYTU3Y2RhYzNmLCBjbGll
bnQgaXMgdXNpbmcgdGhlIGxheW91dCBldmVuIGFmdGVyDQo+ID4+IGxheW91dHJldHVybiwgd2hp
Y2ggSU1ITyBpcyBhIHZpb2xhdGlvbiBvZiByZmM1NjYxLg0KPiA+DQo+ID4gTm8uIEl0IGlzIHVz
aW5nIHRoZSBsYXlvdXRyZXR1cm4gdG8gdGVsbCB0aGUgTURTIHRvIGZlbmNlIG9mZiBJL08gdG8g
YQ0KPiA+IGRhdGEgc2VydmVyIHRoYXQgaXMgbm90IHJlc3BvbmRpbmcuIEl0IGlzbid0IGF0dGVt
cHRpbmcgdG8gdXNlIHRoZQ0KPiA+IGxheW91dCBhZnRlciB0aGUgbGF5b3V0cmV0dXJuOiB0aGUg
d2hvbGUgcG9pbnQgaXMgdGhhdCB3ZSBhcmUgYXR0ZW1wdGluZw0KPiA+IHdyaXRlLXRocm91Z2gt
TURTIGFmdGVyIHRoZSBhdHRlbXB0IHRvIHdyaXRlIHRocm91Z2ggdGhlIERTIHRpbWVkIG91dC4N
Cj4gPg0KPiBCdXQgaXQgaXMgUkZDIHZpb2xhdGlvbiB0aGF0IHRoZXJlIGlzIGluLWZsaWdodCBE
UyBJTyB3aGVuIGNsaWVudA0KPiBzZW5kcyBsYXlvdXRyZXR1cm4sIHJpZ2h0PyBOb3QganVzdCBp
bi1mbGlnaHQsIGNsaWVudCBpcyB3ZWxsIHBvc3NpYmxlDQo+IHRvIHNlbmQgSU8gdG8gRFMgX2Fm
dGVyXyBsYXlvdXRyZXR1cm4gYmVjYXVzZSBzb21lIHRocmVhZCBjYW4gaG9sZA0KPiBsc2VnIHJl
ZmVyZW5jZSBhbmQgbm90IHlldCBzZW5kIElPLg0KDQpPbmNlIHRoZSB3cml0ZSBoYXMgYmVlbiBz
ZW50LCBob3cgZG8geW91IGtub3cgdGhhdCBpdCBpcyBubyBsb25nZXINCidpbi1mbGlnaHQnIHVu
bGVzcyB0aGUgRFMgcmVzcG9uZHM/DQoNCj4gPj4gPj4gQW5kIGl0IGNhdXNlcyBkYXRhIGNvcnJ1
cHRpb24gZm9yIGJsb2NrbGF5b3V0IGlmIGNsaWVudCByZXR1cm5zIGxheW91dA0KPiA+PiA+PiB3
aGlsZSB0aGVyZSBpcyBpbi1mbGlnaHQgZGlzayBJTy4uLg0KPiA+PiA+DQo+ID4+ID4gVGhlbiBp
dCBuZWVkcyB0byB0dXJuIG9mZiBmYXN0IGZhaWxvdmVyIHRvIHdyaXRlLXRocm91Z2gtTURTLg0K
PiA+PiA+DQo+ID4+IElmIHlvdSBzdGlsbCBjb25zaWRlciBpdCBmb2xsb3dpbmcgcmZjNTY2MSwg
SSdkIGNob29zZSB0byBkaXNhYmxlDQo+ID4+IGxheW91dHJldHVybiBpbiBiZWZvcmUgd3JpdGUt
dGhyb3VnaC1NRFMgZm9yIGJsb2NrbGF5b3V0LCBieSBhZGRpbmcNCj4gPj4gc29tZSBmbGFnIGxp
a2UgUE5GU19OT19MQVlPVVRSRVRfT05fRkFMTFRIUlUgc2ltaWxhciB0byBvYmplY3RzJw0KPiA+
PiBQTkZTX0xBWU9VVFJFVF9PTl9TRVRBVFRSLg0KPiA+DQo+ID4gSSBkb24ndCBzZWUgaG93IHRo
YXQgd2lsbCBwcmV2ZW50IGNvcnJ1cHRpb24uDQo+ID4NCj4gPiBJbiBmYWN0LCBJSVJDLCBpbiBO
RlN2NC4yIFNvcmluJ3MgcHJvcG9zZWQgY2hhbmdlcyBzcGVjaWZpY2FsbHkgdXNlIHRoZQ0KPiA+
IGxheW91dHJldHVybiB0byBjb21tdW5pY2F0ZSB0byB0aGUgTURTIHRoYXQgdGhlIERTIGlzIHRp
bWluZyBvdXQgdmlhIGFuDQo+ID4gZXJyb3IgY29kZSAobGlrZSB0aGUgb2JqZWN0IGxheW91dCBo
YXMgZG9uZSBhbGwgdGhlIHRpbWUpLiBIb3cgY2FuIHlvdQ0KPiA+IHJlY29uY2lsZSB0aGF0IGNo
YW5nZSB3aXRoIGEgZmxhZyBzdWNoIGFzIHRoZSBvbmUgeW91IHByb3Bvc2U/DQo+IEkganVzdCBp
bnRlbmQgdG8gdXNlIHRoZSBmbGFnIHRvIGRpc2FibGUgbGF5b3V0cmV0dXJuIGluDQo+IHBuZnNf
bGRfd3JpdGVfZG9uZS4gYmxvY2sgZXh0ZW50cyBhcmUgZGF0YSBhY2Nlc3MgcGVybWlzc2lvbnMg
cGVyDQo+IHJmYzU2NjMuIFdoZW4gd2UgZG9uJ3QgbGF5b3V0cmV0dXJuIGluIHBuZnNfbGRfd3Jp
dGVfZG9uZSgpLCBibG9jaw0KPiBsYXlvdXQgd29ya3MgY29ycmVjdGx5IGJlY2F1c2Ugc2VydmVy
IGNhbiBkZWNpZGUgaWYgdGhlcmUgaXMgZGF0YQ0KPiBhY2Nlc3MgcmFjZSBhbmQgaWYgdGhlcmUg
aXMsIE1EUyBjYW4gcmVjYWxsIHRoZSBsYXlvdXQgZnJvbSBjbGllbnQNCj4gYmVmb3JlIGFwcGx5
aW5nIHRoZSBNRFMgd3JpdGVzLg0KPiANCj4gU29yaW4ncyBwcm9wb3NlZCBlcnJvciBjb2RlIGlz
IGp1c3QgYSBjbGllbnQgaW5kaWNhdGlvbiB0byBzZXJ2ZXIgdGhhdA0KPiB0aGVyZSBpcyBkaXNr
IGFjY2VzcyBlcnJvci4gSXQgaXMgbm90IGludGVuZGVkIHRvIHNvbHZlIHRoZSBkYXRhIHJhY2UN
Cj4gYmV0d2VlbiB3cml0ZS10aHJvdWdoLU1EUyBhbmQgd3JpdGUtdGhyb3VnaC1EUy4NCg0KVGhl
biBob3cgZG8geW91IHNvbHZlIHRoYXQgcmFjZSBvbiBhIGJsb2NrIGRldmljZT8NCg0KLS0gDQpU
cm9uZCBNeWtsZWJ1c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0KDQpOZXRBcHANClRy
b25kLk15a2xlYnVzdEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=
On 2012-08-09 18:39, Myklebust, Trond wrote:
> On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>> <[email protected]> wrote:
>>> On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>>>> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>>>> <[email protected]> wrote:
>>>>> Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>>>>> disconnected data server) we've been sending layoutreturn calls
>>>>> while there is potentially still outstanding I/O to the data
>>>>> servers. The reason we do this is to avoid races between replayed
>>>>> writes to the MDS and the original writes to the DS.
>>>>>
>>>>> When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>>>>> be triggered because it assumes that we would never call
>>>>> layoutreturn without knowing that all I/O to the DS is
>>>>> finished. The fix is to remove the BUG_ON() now that the
>>>>> assumptions behind the test are obsolete.
>>>>>
>>>> Isn't MDS supposed to recall the layout if races are possible between
>>>> outstanding write-to-DS and write-through-MDS?
>>>
>>> Where do you read that in RFC5661?
>>>
>> That's my (maybe mis-)understanding of how server works... But looking
>> at rfc5661 section 18.44.3. layoutreturn implementation.
>> "
>> After this call,
>> the client MUST NOT use the returned layout(s) and the associated
>> storage protocol to access the file data.
>> "
>> And given commit 0a57cdac3f, client is using the layout even after
>> layoutreturn, which IMHO is a violation of rfc5661.
>
> No. It is using the layoutreturn to tell the MDS to fence off I/O to a
> data server that is not responding. It isn't attempting to use the
> layout after the layoutreturn: the whole point is that we are attempting
> write-through-MDS after the attempt to write through the DS timed out.
>
I hear you, but this use case is valid after a time out / disconnect
(which will translate to PNFS_OSD_ERR_UNREACHABLE for the objects layout)
In other cases, I/Os to the DS might obviously be in flight and the BUG_ON
indicates that.
IMO, the right way to implement that is to initially mark the lsegs invalid
and increment plh_block_lgets, as we do today in _pnfs_return_layout
but actually send the layout return only when the last segment is dereferenced.
Benny
T24gU3VuLCAyMDEyLTA4LTEyIGF0IDIwOjM2ICswMzAwLCBCb2F6IEhhcnJvc2ggd3JvdGU6DQo+
IE9uIDA4LzA5LzIwMTIgMDY6MzkgUE0sIE15a2xlYnVzdCwgVHJvbmQgd3JvdGU6DQo+ID4gSWYg
dGhlIHByb2JsZW0gaXMgdGhhdCB0aGUgRFMgaXMgZmFpbGluZyB0byByZXNwb25kLCBob3cgZG9l
cyB0aGUgY2xpZW50DQo+ID4ga25vdyB0aGF0IHRoZSBpbi1mbGlnaHQgSS9PIGhhcyBlbmRlZD8N
Cj4gDQo+IEZvciB0aGUgY2xpZW50LCB0aGUgYWJvdmUgRFMgaW4gcXVlc3Rpb24sIGhhcyB0aW1l
ZC1vdXQsIHdlIGhhdmUgcmVzZXQNCj4gaXQncyBzZXNzaW9uIGFuZCBjbG9zZWQgaXQncyBzb2Nr
ZXRzLiBBbmQgYWxsIGl0J3MgUlBDIHJlcXVlc3RzIGhhdmUNCj4gYmVlbiwgb3IgYXJlIGJlaW5n
LCBlbmRlZCB3aXRoIGEgdGltZW91dC1lcnJvci4gU28gdGhlIHRpbWVkLW91dA0KPiBEUyBpcyBh
IG5vLW9wLiBBbGwgaXQncyBJTyByZXF1ZXN0IHdpbGwgZW5kIHZlcnkgc29vbiwgaWYgbm90IGFs
cmVhZHkuDQo+IA0KPiBBIERTIHRpbWUtb3V0IGlzIGp1c3QgYSB2ZXJ5IHZhbGlkLCBhbmQgbWVh
bmluZ2Z1bCByZXNwb25zZSwganVzdCBsaWtlDQo+IGFuIG9wLWRvbmUtd2l0aC1lcnJvci4gVGhp
cyB3YXMgd2hhdCBBbmR5IGFkZGVkIHRvIHRoZSBSRkMncyBlcnJhdGENCj4gd2hpY2ggSSBhZ3Jl
ZSB3aXRoLg0KPiANCj4gPiANCj4gPiBOby4gSXQgaXMgdXNpbmcgdGhlIGxheW91dHJldHVybiB0
byB0ZWxsIHRoZSBNRFMgdG8gZmVuY2Ugb2ZmIEkvTyB0byBhDQo+ID4gZGF0YSBzZXJ2ZXIgdGhh
dCBpcyBub3QgcmVzcG9uZGluZy4gSXQgaXNuJ3QgYXR0ZW1wdGluZyB0byB1c2UgdGhlDQo+ID4g
bGF5b3V0IGFmdGVyIHRoZSBsYXlvdXRyZXR1cm46IA0KPiANCj4gPiB0aGUgd2hvbGUgcG9pbnQg
aXMgdGhhdCB3ZSBhcmUgYXR0ZW1wdGluZw0KPiA+IHdyaXRlLXRocm91Z2gtTURTIGFmdGVyIHRo
ZSBhdHRlbXB0IHRvIHdyaXRlIHRocm91Z2ggdGhlIERTIHRpbWVkIG91dC4NCj4gPiANCj4gDQo+
IFRyb25kIFNUT1AhISEgdGhpcyBpcyBwdXJlIGJ1bGxzaGl0LiBZb3UgZ3V5cyB0b29rIHRoZSBv
cHBvcnR1bml0eSBvZg0KPiBtZSBiZWluZyBpbiBIb3NwaXRhbCwgYW5kIHRoZSByZXN0IG9mIHRo
ZSBidW5jaCBub3QgaGF2aW5nIGEgY2x1ZS4gQW5kDQo+IHNudWNrIGluIGEgcGF0Y2ggdGhhdCBp
cyB0b3RhbGx5IHdyb25nIGZvciBldmVyeW9uZSwgbm90IHRha2luZyBjYXJlIG9mDQo+IGFueSBv
dGhlciBMRCAqY3Jhc2hlcyogLiBBbmQgZXNwZWNpYWxseSB3aGVuIHRoaXMgcGF0Y2ggaXMgd3Jv
bmcgZXZlbiBmb3INCj4gZmlsZXMgbGF5b3V0Lg0KPiANCj4gVGhpcyBhYm92ZSBoZXJlIGlzIHdo
ZXJlIHlvdSBhcmUgd3JvbmchISBZb3UgZG9uJ3QgdW5kZXJzdGFuZCBteSBwb2ludCwNCj4gYW5k
IGlnbm9yZSBteSBjb21tZW50cy4gU28gbGV0IG1lIHN0YXRlIGl0IGFzIGNsZWFyIGFzIEkgY2Fu
Lg0KDQpZT1UgYXJlIGlnbm9yaW5nIHRoZSByZWFsaXR5IG9mIFN1blJQQy4gVGhlcmUgaXMgbm8g
YWJvcnQvY2FuY2VsL3RpbWVvdXQNCmZvciBhbiBSUEMgY2FsbCBvbmNlIGl0IGhhcyBzdGFydGVk
LiBUaGlzIGlzIHdoeSB3ZSBuZWVkIGZlbmNpbmcNCl9zcGVjaWZpY2FsbHlfIGZvciB0aGUgcE5G
UyBmaWxlcyBjbGllbnQuDQoNCj4gKExldHMgYXNzdW1lIGZpbGVzIGxheW91dCwgZm9yIGJsb2Nr
cyBhbmQgb2JqZWN0cyBpdCdzIGEgYml0IGRpZmZlcmVudA0KPiAgYnV0IG1vc3RseSB0aGUgc2Ft
ZS4pDQoNClRoYXQsIGFuZCB0aGUgZmFjdCB0aGF0IGZlbmNpbmcgaGFzbid0IGJlZW4gaW1wbGVt
ZW50ZWQgZm9yIGJsb2NrcyBhbmQNCm9iamVjdHMuIFRoZSBjb21taXQgaW4gcXVlc3Rpb24gaXMg
ODJjN2M3YTVhIChORlN2NC4xIHJldHVybiB0aGUgTEFZT1VUDQpmb3IgZWFjaCBmaWxlIHdpdGgg
ZmFpbGVkIERTIGNvbm5lY3Rpb24gSS9PKSBhbmQgdG91Y2hlcyBvbmx5DQpmcy9uZnMvbmZzNGZp
bGVsYXlvdXQuYy4gSXQgY2Fubm90IGJlIGFmZmVjdGluZyBibG9ja3MgYW5kIG9iamVjdHMuDQoN
Cj4gLSBIZWF2eSBJTyBpcyBnb2luZyBvbiwgdGhlIGRldmljZV9pZCBpbiBxdWVzdGlvbiBoYXMg
KjMqIERTcyBpbiBpdCdzDQo+ICAgZGV2aWNlIHRvcG9ncmFwaHkuIFNheSBEUzEsIERTMiwgRFMz
DQo+IA0KPiAtIFdlIGhhdmUgYmVlbiBxdWV1aW5nIElPLCBhbmQgYWxsIHF1ZXVlcyBhcmUgZnVs
bC4gKHdlIGhhdmUgMyBxdWV1ZXMgaW4NCj4gICBpbiBxdWVzdGlvbiwgcmlnaHQ/IFdoYXQgaXMg
dGhlIG1heGltdW0gUSBkZXB0aCBwZXIgZmlsZXMtRFM/IEkga25vdw0KPiAgIHRoYXQgaW4gYmxv
Y2tzIGFuZCBvYmplY3RzIHdlIHVzdWFsbHkgaGF2ZSwgSSB0aGluaywgc29tZXRoaW5nIGxpa2Ug
MTI4Lg0KPiAgIFRoaXMgaXMgYSAqdHVuYWJsZSogaW4gdGhlIGJsb2NrLWxheWVyJ3MgcmVxdWVz
dC1xdWV1ZS4gSXMgaXQgbm90IHNvbWUNCj4gICBuZWdvdGlhdGVkIHBhcmFtZXRlciB3aXRoIHRo
ZSBORlMgc2VydmVycz8pDQo+IA0KPiAtIE5vdywgYm9vbSBEUzIgaGFzIHRpbWVkLW91dC4gVGhl
IExpbnV4LWNsaWVudCByZXNldHMgdGhlIHNlc3Npb24gYW5kDQo+ICAgaW50ZXJuYWxseSBjbG9z
ZXMgYWxsIHNvY2tldHMgb2YgdGhhdCBzZXNzaW9uLiBBbGwgdGhlIFJQQ3MgdGhhdA0KPiAgIGJl
bG9uZyB0byBEUzIgYXJlIGJlaW5nIHJldHVybmVkIHVwIHdpdGggYSB0aW1lb3V0IGVycm9yLiBU
aGlzIG9uZQ0KPiAgIGlzIGp1c3QgdGhlIGZpcnN0IG9mIGFsbCB0aG9zZSBiZWxvbmdpbmcgdG8g
dGhpcyBEUzIuIFRoZXkgd2lsbA0KPiAgIGJlIGRlY3JlbWVudGluZyB0aGUgcmVmZXJlbmNlIGZv
ciB0aGlzIGxheW91dCB2ZXJ5LCB2ZXJ5IHNvb24uDQo+IA0KPiAtIEJ1dCB3aGF0IGFib3V0IERT
MSwgYW5kIERTMyBSUENzLiBXaGF0IHNob3VsZCB3ZSBkbyB3aXRoIHRob3NlPw0KPiAgIFRoaXMg
aXMgd2hlcmUgeW91IGd1eXMgKFRyb25kIGFuZCBBbmR5KSBhcmUgd3JvbmcuIFdlIG11c3QgYWxz
bw0KPiAgIHdhaXQgZm9yIHRoZXNlIFJQQydzIGFzIHdlbGwuIEFuZCBvcHBvc2l0ZSB0byB3aGF0
IHlvdSB0aGluaywgdGhpcw0KPiAgIHNob3VsZCBub3QgdGFrZSBsb25nLiBMZXQgbWUgZXhwbGFp
bjoNCj4gDQo+ICAgV2UgZG9uJ3Qga25vdyBhbnl0aGluZyBhYm91dCBEUzEgYW5kIERTMywgZWFj
aCBtaWdodCBiZSwgZWl0aGVyLA0KPiAgICJIYXZpbmcgdGhlIHNhbWUgY29tbXVuaWNhdGlvbiBw
cm9ibGVtLCBsaWtlIERTMiIuIE9yICJpcyBqdXN0IHdvcmtpbmcNCj4gICBmaW5lIi4gU28gbGV0
cyBzYXkgZm9yIGV4YW1wbGUgdGhhdCBEUzMgd2lsbCBhbHNvIHRpbWUtb3V0IGluIHRoZQ0KPiAg
IGZ1dHVyZSwgYW5kIHRoYXQgRFMxIGlzIGp1c3QgZmluZSBhbmQgaXMgd3JpdGluZyBhcyB1c3Vh
bC4NCj4gDQo+ICAgKiBEUzEgLSBTaW5jZSBpdCdzIHdvcmtpbmcsIGl0IGhhcyBtb3N0IHByb2Jh
Ymx5IGFscmVhZHkgZG9uZQ0KPiAgICAgd2l0aCBhbGwgaXQncyBJTywgYmVjYXVzZSB0aGUgTkZT
IHRpbWVvdXQgaXMgdXN1YWxseSBtdWNoIGxvbmdlcg0KPiAgICAgdGhlbiB0aGUgbm9ybWFsIFJQ
QyB0aW1lLCBhbmQgc2luY2Ugd2UgYXJlIHF1ZXVpbmcgZXZlbmx5IG9uDQo+ICAgICBhbGwgMyBE
U3MsIGF0IHRoaXMgcG9pbnQgbXVzdCBwcm9iYWJseSwgYWxsIG9mIERTMSBSUENzIGFyZQ0KPiAg
ICAgYWxyZWFkeSBkb25lLiAoQW5kIGxheW91dCBoYXMgYmVlbiBkZS1yZWZlcmVuY2VkKS4NCj4g
DQo+ICAgKiBEUzMgLSBXaWxsIHRpbWVvdXQgaW4gdGhlIGZ1dHVyZSwgd2hlbiB3aWxsIHRoYXQg
YmU/DQo+ICAgICBTbyBsZXQgbWUgc3RhcnQgd2l0aCwgc2F5aW5nOg0KPiAgICAgKDEpLiBXZSBj
b3VsZCBlbmhhbmNlIG91ciBjb2RlIGFuZCBwcm9hY3RpdmVseSwgDQo+ICAgICAgICAgImNhbmNl
bC9hYm9ydCIgYWxsIFJQQ3MgdGhhdCBiZWxvbmcgdG8gRFMzIChtb3JlIG9uIHRoaXMNCj4gICAg
ICAgICAgYmVsb3cpDQoNCldoaWNoIG1ha2VzIHRoZSByYWNlIF9XT1JTRV8uIEFzIEkgc2FpZCBh
Ym92ZSwgdGhlcmUgaXMgbm8gJ2NhbmNlbCBSUEMnDQpvcGVyYXRpb24gaW4gU1VOUlBDLiBPbmNl
IHlvdXIgUlBDIGNhbGwgaXMgbGF1bmNoZWQsIGl0IGNhbm5vdCBiZQ0KcmVjYWxsZWQuIEFsbCB5
b3VyIGRpc2N1c3Npb24gYWJvdmUgaXMgYWJvdXQgdGhlIGNsaWVudCBzaWRlLCBhbmQNCmlnbm9y
ZXMgd2hhdCBtYXkgYmUgaGFwcGVuaW5nIG9uIHRoZSBkYXRhIHNlcnZlciBzaWRlLiBUaGUgZmVu
Y2luZyBpcw0Kd2hhdCBpcyBuZWVkZWQgdG8gZGVhbCB3aXRoIHRoZSBkYXRhIHNlcnZlciBwaWN0
dXJlLg0KDQo+ICAgICAoMikuIE9yIFdlIGNhbiBwcm92ZSB0aGF0IERTMydzIFJQQ3Mgd2lsbCB0
aW1lb3V0IGF0IHdvcnN0DQo+ICAgICAgICAgIGNhc2UgMSB4IE5GUy10aW1lb3V0IGFmdGVyIGFi
b3ZlIERTMiB0aW1lb3V0IGV2ZW50LCBvcg0KPiAgICAgICAgICAyIHggTkZTLXRpbWVvdXQgYWZ0
ZXIgdGhlIHF1ZXVpbmcgb2YgdGhlIGZpcnN0IHRpbWVkLW91dA0KPiAgICAgICAgICBSUEMuIEFu
ZCBzdGF0aXN0aWNhbGx5IGluIHRoZSBhdmVyYWdlIGNhc2UgRFMzIHdpbGwgdGltZW91dA0KPiAg
ICAgICAgICB2ZXJ5IG5lYXIgdGhlIHRpbWUgRFMyIHRpbWVkLW91dC4NCj4gDQo+ICAgICAgICAg
IFRoaXMgaXMgZWFzeSBzaW5jZSB0aGUgbGFzdCBJTyB3ZSBxdWV1ZWQgd2FzIHRoZSBvbmUgdGhh
dA0KPiAgICAgICAgICBtYWRlIERTMidzIHF1ZXVlIHRvIGJlIGZ1bGwsIGFuZCBpdCB3YXMga2Vw
dCBmdWxsIGJlY2F1c2UNCj4gICAgICAgICAgRFMyIHN0b3BwZWQgcmVzcG9uZGluZyBhbmQgbm90
aGluZyBlbXB0aWVkIHRoZSBxdWV1ZS4NCj4gDQo+ICAgICAgU28gdGhlIGVhc2llc3Qgd2UgY2Fu
IGRvIGlzIHdhaXQgZm9yIERTMyB0byB0aW1lb3V0LCBzb29uDQo+ICAgICAgZW5vdWdoLCBhbmQg
b25jZSB0aGF0IHdpbGwgaGFwcGVuLCBzZXNzaW9uIHdpbGwgYmUgcmVzZXQgYW5kIGFsbA0KPiAg
ICAgIFJQQ3Mgd2lsbCBlbmQgd2l0aCBhbiBlcnJvci4NCg0KDQpZb3UgYXJlIHN0aWxsIG9ubHkg
ZGlzY3Vzc2luZyB0aGUgY2xpZW50IHNpZGUuDQoNClJlYWQgbXkgbGlwczogU3VuIFJQQyBPUEVS
QVRJT05TIERPIE5PVCBUSU1FT1VUIEFORCBDQU5OT1QgQkUgQUJPUlRFRCBPUg0KQ0FOQ0VMRUQu
IEZlbmNpbmcgaXMgdGhlIGNsb3Nlc3Qgd2UgY2FuIGNvbWUgdG8gYW4gYWJvcnQgb3BlcmF0aW9u
Lg0KDQo+IFNvIGluIHRoZSB3b3JzdCBjYXNlIHNjZW5hcmlvIHdlIGNhbiByZWNvdmVyIDIgeCBO
RlMtdGltZW91dCBhZnRlcg0KPiBhIG5ldHdvcmsgcGFydGl0aW9uLCB3aGljaCBpcyBqdXN0IDEg
eCBORlMtdGltZW91dCwgYWZ0ZXIgeW91cg0KPiBzY2hpem9waHJlbmljIEZFTkNFX01FX09GRiwg
bmV3bHkgaW52ZW50ZWQgb3BlcmF0aW9uLg0KPiANCj4gV2hhdCB3ZSBjYW4gZG8gdG8gZW5oYW5j
ZSBvdXIgY29kZSB0byByZWR1Y2UgZXJyb3IgcmVjb3ZlcnkgdG8NCj4gMSB4IE5GUy10aW1lb3V0
Og0KPiANCj4gLSBEUzMgYWJvdmU6DQo+ICAgKEFzIEkgc2FpZCBEUzEncyBxdWV1ZXMgYXJlIG5v
dyBlbXB0eSwgYmVjYXVzZSBpdCB3YXMgd29ya2luZyBmaW5lLA0KPiAgICBTbyBEUzMgaXMgYSBy
ZXByZXNlbnRhdGlvbiBvZiBhbGwgRFMncyB0aGF0IGhhdmUgUlBDcyBhdCB0aGUNCj4gICAgdGlt
ZSBEUzIgdGltZWQtb3V0LCB3aGljaCBiZWxvbmcgdG8gdGhpcyBsYXlvdXQpDQo+IA0KPiAgIFdl
IGNhbiBwcm9hY3RpdmVseSBhYm9ydCBhbGwgUlBDcyBiZWxvbmdpbmcgdG8gRFMzLiBJZiB0aGVy
ZSBpcw0KPiAgIGEgd2F5IHRvIGludGVybmFsbHkgYWJvcnQgUlBDJ3MgdXNlIHRoYXQuIEVsc2Ug
anVzdCByZXNldCBpdCdzDQo+ICAgc2Vzc2lvbiBhbmQgYWxsIHNvY2tldHMgd2lsbCBjbG9zZSAo
YW5kIHJlb3BlbiksIGFuZCBhbGwgUlBDJ3MNCj4gICB3aWxsIGVuZCB3aXRoIGEgZGlzY29ubmVj
dCBlcnJvci4NCg0KTm90IG9uIG1vc3Qgc2VydmVycyB0aGF0IEknbSBhd2FyZSBvZi4gSWYgeW91
IGNsb3NlIG9yIHJlc2V0IHRoZSBzb2NrZXQNCm9uIHRoZSBjbGllbnQsIHRoZW4gdGhlIExpbnV4
IHNlcnZlciB3aWxsIGhhcHBpbHkgY29udGludWUgdG8gcHJvY2Vzcw0KdGhvc2UgUlBDIGNhbGxz
OyBpdCBqdXN0IHdvbid0IGJlIGFibGUgdG8gc2VuZCBhIHJlcGx5Lg0KRnVydGhlcm1vcmUsIGlm
IHRoZSBwcm9ibGVtIGlzIHRoYXQgdGhlIGRhdGEgc2VydmVyIGlzbid0IHJlc3BvbmRpbmcsDQp0
aGVuIGEgc29ja2V0IGNsb3NlL3Jlc2V0IHRlbGxzIHlvdSBub3RoaW5nIGVpdGhlci4NCg0KPiAt
IEJvdGggRFMyIHRoYXQgdGltZWQtb3V0LCBhbmQgRFMzIHRoYXQgd2FzIGFib3J0ZWQuIFNob3Vs
ZCBiZQ0KPiAgIG1hcmtlZCB3aXRoIGEgZmxhZy4gV2hlbiBuZXcgSU8gdGhhdCBiZWxvbmcgdG8g
c29tZSBvdGhlcg0KPiAgIGlub2RlIHRocm91Z2ggc29tZSBvdGhlciBsYXlvdXQrZGV2aWNlX2lk
IGVuY291bnRlcnMgYSBmbGFnZ2VkDQo+ICAgZGV2aWNlLCBpdCBzaG91bGQgYWJvcnQgYW5kIHR1
cm4gdG8gTURTIElPLCB3aXRoIGFsc28gaW52YWxpZGF0aW5nDQo+ICAgaXQncyBsYXlvdXQsIGFu
ZCBoZW5zLCBzb29uIGVub3VnaCB0aGUgZGV2aWNlX2lkIGZvciBEUzImMyB3aWxsIGJlDQo+ICAg
ZGUtcmVmZXJlbmNlZCBhbmQgYmUgcmVtb3ZlZCBmcm9tIGRldmljZSBjYWNoZS4gKEFuZCBhbGwg
cmVmZXJlbmNpbmcNCj4gICBsYXlvdXRzIGFyZSBub3cgZ29uZSkNCg0KVGhlcmUgaXMgbm8gUlBD
IGFib3J0IGZ1bmN0aW9uYWxpdHkgU3VuIFJQQy4gQWdhaW4sIHRoaXMgYXJndW1lbnQgcmVsaWVz
DQpvbiBmdW5jdGlvbmFsaXR5IHRoYXQgX2RvZXNuJ3RfIGV4aXN0Lg0KDQo+ICAgU28gd2UgZG8g
bm90IGNvbnRpbnVlIHF1ZXVpbmcgbmV3IElPIHRvIGRlYWQgZGV2aWNlcy4gQW5kIHNpbmNlIG1v
c3QNCj4gICBwcm9iYWJseSBNRFMgd2lsbCBub3QgZ2l2ZSB1cyBkZWFkIHNlcnZlcnMgaW4gbmV3
IGxheW91dCwgd2Ugc2hvdWxkIGJlDQo+ICAgZ29vZC4NCj4gSW4gc3VtbWVyeS4NCj4gLSBGRU5D
RV9NRV9PRkYgaXMgYSBuZXcgb3BlcmF0aW9uLCBhbmQgaXMgbm90ID09PSBMQVlPVVRfUkVUVVJO
LiBDbGllbnQNCj4gICAqbXVzdCBub3QqIHNrYi1zZW5kIGEgc2luZ2xlIGJ5dGUgYmVsb25naW5n
IHRvIGEgbGF5b3V0LCBhZnRlciB0aGUgc2VuZA0KPiAgIG9mIExBWU9VVF9SRVRVUk4uDQo+ICAg
KEl0IG5lZWQgbm90IHdhaXQgZm9yIE9QVF9ET05FIGZyb20gRFMgdG8gZG8gdGhhdCwgaXQganVz
dCBtdXN0IG1ha2UNCj4gICAgc3VyZSwgdGhhdCBhbGwgaXQncyBpbnRlcm5hbCwgb3Igb24tdGhl
LXdpcmUgcmVxdWVzdCwgYXJlIGFib3J0ZWQNCj4gICAgYnkgZWFzaWx5IGNsb3NpbmcgdGhlIHNv
Y2tldHMgdGhleSBiZWxvbmcgdG9vLCBhbmQvb3Igd2FpdGluZyBmb3INCj4gICAgaGVhbHRoeSBE
UydzIElPIHRvIGJlIE9QVF9ET05FIC4gU28gdGhlIGNsaWVudCBpcyBub3QgZGVwZW5kZW50IG9u
DQo+ICAgIGFueSBEUyByZXNwb25zZSwgaXQgaXMgb25seSBkZXBlbmRlbnQgb24gaXQncyBpbnRl
cm5hbCBzdGF0ZSBiZWluZw0KPiAgICAqY2xlYW4qIGZyb20gYW55IG1vcmUgc2tiLXNlbmQocykp
DQoNCkRpdHRvDQoNCj4gLSBUaGUgcHJvcGVyIGltcGxlbWVudGF0aW9uIG9mIExBWU9VVF9SRVRV
Uk4gb24gZXJyb3IgZm9yIGZhc3QgdHVybm92ZXINCj4gICBpcyBub3QgaGFyZCwgYW5kIGRvZXMg
bm90IGludm9sdmUgYSBuZXcgaW52ZW50ZWQgTkZTIG9wZXJhdGlvbiBzdWNoDQo+ICAgYXMgRkVO
Q0VfTUVfT0ZGLiBQcm9wZXIgY29kZGVkIGNsaWVudCwgaW5kZXBlbmRlbnRseSwgd2l0aG91dA0K
PiAgIHRoZSBhaWQgb2YgYW55IEZFTkNFX01FX09GRiBvcGVyYXRpb24sIGNhbiBhY2hpZXZlIGEg
ZmFzdGVyIHR1cm5hcm91bmQNCj4gICBieSBhY3RpdmVseSByZXR1cm5pbmcgYWxsIGxheW91dHMg
dGhhdCBiZWxvbmcgdG8gYSBiYWQgRFMsIGFuZCBub3QNCj4gICB3YWl0aW5nIGZvciBhIGZlbmNl
LW9mZiBvZiBhIHNpbmdsZSBsYXlvdXQsIHRoZW4gZW5jb3VudGVyaW5nIGp1c3QNCj4gICB0aGUg
c2FtZSBlcnJvciB3aXRoIGFsbCBvdGhlciBsYXlvdXRzIHRoYXQgaGF2ZSB0aGUgc2FtZSBEUyAg
ICAgDQoNCldoYXQgZG8geW91IG1lYW4gYnkgImFsbCBsYXlvdXRzIHRoYXQgYmVsb25nIHRvIGEg
YmFkIERTIj8gTGF5b3V0cyBkb24ndA0KYmVsb25nIHRvIGEgRFMsIGFuZCBzbyB0aGVyZSBpcyBu
byB3YXkgdG8gZ2V0IGZyb20gYSBEUyB0byBhIGxheW91dC4NCg0KPiAtIEFuZCBJIGtub3cgdGhh
dCBqdXN0IGFzIHlvdSBkaWQgbm90IHJlYWQgbXkgZW1haWxzIGZyb20gYmVmb3JlDQo+ICAgbWUg
Z29pbmcgdG8gSG9zcGl0YWwsIHlvdSB3aWxsIGNvbnRpbnVlIHRvIG5vdCB1bmRlcnN0YW5kIHRo
aXMNCj4gICBvbmUsIG9yIHdoYXQgSSdtIHRyeWluZyB0byBleHBsYWluLCBhbmQgd2lsbCBtb3N0
IHByb2JhYmx5IGlnbm9yZQ0KPiAgIGFsbCBvZiBpdC4gQnV0IHBsZWFzZSBub3RlIG9uZSB0aGlu
ZzoNCg0KSSByZWFkIHRoZW0sIGJ1dCBqdXN0IGFzIG5vdywgdGhleSBjb250aW51ZSB0byBpZ25v
cmUgdGhlIHJlYWxpdHkgYWJvdXQNCnRpbWVvdXRzOiB0aW1lb3V0cyBtZWFuIF9ub3RoaW5nXyBp
biBhbiBSUEMgZmFpbG92ZXIgc2l0dWF0aW9uLiBUaGVyZSBpcw0Kbm8gUlBDIGFib3J0IGZ1bmN0
aW9uYWxpdHkgdGhhdCB5b3UgY2FuIHJlbHkgb24gb3RoZXIgdGhhbiBmZW5jaW5nLg0KDQo+ICAg
ICBZT1UgaGF2ZSBzYWJvdGFnZWQgdGhlIE5GUyA0LjEgTGludXggY2xpZW50LCB3aGljaCBpcyBu
b3cgdG90YWxseQ0KPiAgICAgbm90IFNURCBjb21wbGFpbnQsIGFuZCBoYXZlIGludHJvZHVjZWQg
Q1JBU0hzLiBBbmQgZm9yIG5vIGdvb2QNCj4gICAgIHJlYXNvbi4NCg0KU2VlIGFib3ZlLg0KDQot
LSANClRyb25kIE15a2xlYnVzdA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFw
cA0KVHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQoNCg==
T24gVHVlLCAyMDEyLTA4LTE0IGF0IDAyOjM5ICswMzAwLCBCb2F6IEhhcnJvc2ggd3JvdGU6DQo+
IE9uIDA4LzEzLzIwMTIgMDc6MjYgUE0sIE15a2xlYnVzdCwgVHJvbmQgd3JvdGU6DQo+IA0KPiA+
PiBUaGlzIGFib3ZlIGhlcmUgaXMgd2hlcmUgeW91IGFyZSB3cm9uZyEhIFlvdSBkb24ndCB1bmRl
cnN0YW5kIG15IHBvaW50LA0KPiA+PiBhbmQgaWdub3JlIG15IGNvbW1lbnRzLiBTbyBsZXQgbWUg
c3RhdGUgaXQgYXMgY2xlYXIgYXMgSSBjYW4uDQo+ID4gDQo+ID4gWU9VIGFyZSBpZ25vcmluZyB0
aGUgcmVhbGl0eSBvZiBTdW5SUEMuIFRoZXJlIGlzIG5vIGFib3J0L2NhbmNlbC90aW1lb3V0DQo+
ID4gZm9yIGFuIFJQQyBjYWxsIG9uY2UgaXQgaGFzIHN0YXJ0ZWQuIFRoaXMgaXMgd2h5IHdlIG5l
ZWQgZmVuY2luZw0KPiA+IF9zcGVjaWZpY2FsbHlfIGZvciB0aGUgcE5GUyBmaWxlcyBjbGllbnQu
DQo+ID4gDQo+IA0KPiANCj4gQWdhaW4gd2UgaGF2ZSBhIGNvbW11bmljYXRpb24gcHJvYmxlbSBi
ZXR3ZWVuIHVzLiBJIHNheSBzb21lIHdvcmRzIGFuZA0KPiBtZWFuIG9uZSB0aGluZywgYW5kIHlv
dSBzYXkgYW5kIGhlYXIgdGhlIHNhbWUgd29yZHMgYnV0IGF0dGFjaCBkaWZmZXJlbnQNCj4gbWVh
bmluZ3MgdG8gdGhlbS4gVGhpcyBpcyBubyBvbmUncyBmYXVsdCBpdCdzIGp1c3QgaXMuDQo+IA0K
PiBMZXRzIGRvIGFuIGV4cGVyaW1lbnQsIG1vdW50IGEgcmVndWxhciBORlM0IGluIC1vIHNvZnQg
bW9kZSBhbmQgc3RhcnQNCj4gd3JpdGluZyB0byBzZXJ2ZXIsIHNheSB3aXRoIGRkLiBOb3cgZGlz
Y29ubmVjdCB0aGUgY2FibGUuIEFmdGVyIHNvbWUgdGltZW91dA0KPiB0aGUgZGQgd2lsbCByZXR1
cm4gd2l0aCAiSU8gZXJyb3IiLCBhbmQgd2lsbCBzdG9wIHdyaXRpbmcgdG8gZmlsZS4NCj4gDQo+
IFRoaXMgaXMgdGhlIHRpbWVvdXQgSSBtZWFuLiBTdXJlbHkgc29tZSBSUEMtcmVxdWVzdHMgZGlk
IG5vdCBjb21wbGV0ZSBhbmQNCj4gcmV0dXJuZWQgdG8gTkZTIGNvcmUgd2l0aCBzb21lIGtpbmQg
b2YgZXJyb3IuDQo+IA0KPiBXaXRoIFJQQy1yZXF1ZXN0cyBJIGRvIG5vdCBtZWFuIHRoZSBSUEMg
cHJvdG9jb2wgb24gdGhlIHdpcmUsIEkgbWVhbiB0aGF0DQo+IGVudGl0eSBpbnNpZGUgdGhlIExp
bnV4IEtlcm5lbCB3aGljaCByZXByZXNlbnRzIGFuIFJQQy4gU3VybHkgc29tZQ0KPiBsaW51eC1S
UEMtcmVxdWVzdHMgb2JqZWN0cyB3ZXJlIG5vdCByZWxlYXNlZCBkbyB0byBhIHNlcnZlciAicnBj
LWRvbmUiDQo+IHJlY2VpdmVkLiBCdXQgZG8gdG8gYW4gaW50ZXJuYWwgbWVjaGFuaXNtIHRoYXQg
Y2FsbGVkIHRoZSAicmVsZWFzZSIgbWV0aG9kIGRvIHRvDQo+IGEgY29tbXVuaWNhdGlvbiB0aW1l
b3V0Lg0KPiANCj4gU28gdGhpcyBpcyB3aGF0IEkgY2FsbCAicmV0dXJuZWQgd2l0aCBhIHRpbWVv
dXQiLiBJdCBkb2VzIGV4aXN0IGFuZCB1c2VkDQo+IGV2ZXJ5IGRheS4NCj4gDQo+IEV2ZW4gYmV0
dGVyIGlmIEkgZG9uJ3QgZGlzY29ubmVjdCB0aGUgd2lyZSBidXQgZG8gYW4gaWZfZG93biBvciBo
YWx0IG9uIHRoZQ0KPiBzZXJ2ZXIsIHRoZSBkZCdzIElPIGVycm9yIHdpbGwgaGFwcGVuIGltbWVk
aWF0ZWx5LCBub3QgZXZlbiB3YWl0IGZvcg0KPiBhbnkgdGltZW91dC4gVGhpcyBpcyBiZWNhdXNl
IHRoZSBzb2NrZXQgaXMgb3JkZXJseSBjbG9zZWQgYW5kIGFsbA0KPiBzZW5kcy9yZWNlaXZlcyB3
aWxsIHJldHVybiBxdWlja2x5IHdpdGggYSAiZGlzY29ubmVjdC1lcnJvciIuDQo+IA0KPiBXaGVu
IEkgdXNlIGEgc2luZ2xlIFNlcnZlciBsaWtlIHRoZSBuZnM0IGFib3ZlLiBUaGVuIHRoZXJlIGlz
IG9uZSBmYWN0DQo+IGluIGFib3ZlIHNjZW5hcmlvIHRoYXQgSSB3YW50IHRvIHBvaW50IG91dDoN
Cj4gDQo+ICAgICBBdCBzb21lIHBvaW50IGluIHRoZSBORlMtQ29yZSBzdGF0ZS4gVGhlcmUgaXMg
YSBwb2ludCB0aGF0IG5vIG1vcmUNCj4gICAgIHJlcXVlc3RzIGFyZSBpc3N1ZWQsIGFsbCBvbGQg
cmVxdWVzdCBoYXZlIHJlbGVhc2VkLCBhbmQgYW4gZXJyb3IgaXMNCj4gICAgIHJldHVybmVkIHRv
IHRoZSBhcHBsaWNhdGlvbi4gQXQgdGhhdCBwb2ludCB0aGUgY2xpZW50IHdpbGwgbm90IGNhbGwN
Cj4gICAgIHNrYi1zZW5kLCBhbmQgd2lsbCBub3QgdHJ5IGZhcnRoZXIgY29tbXVuaWNhdGlvbiB3
aXRoIHRoZSBTZXJ2ZXIuDQo+IA0KPiBUaGlzIGlzIHdoYXQgbXVzdCBoYXBwZW4gd2l0aCBBTEwg
RFNzIHRoYXQgYmVsb25nIHRvIGEgbGF5b3V0LCBiZWZvcmUNCj4gY2xpZW50IHNob3VsZCBiZSBM
QVlPVVRfUkVUVVJOKGluZykuIFRoZSBjbGllbnQgY2FuIG9ubHkgZG8gaXQncyBqb2IuIFRoYXQg
aXM6DQo+IA0KPiAgICBTVE9QIGFueSBza2Itc2VuZCwgdG8gYW55IG9mIHRoZSBEU3MgaW4gYSBs
YXlvdXQuDQo+ICAgIE9ubHkgdGhlbiBpdCBpcyBjb21wbHlpbmcgdG8gdGhlIFJGQy4NCj4gDQo+
IFNvIHRoaXMgaXMgd2hhdCBJIG1lYW4gYnkgInJldHVybiB3aXRoIGEgdGltZW91dCBiZWxvdyIN
Cg0KDQpJIGhlYXIgeW91LCBub3cgbGlzdGVuIHRvIG1lLg0KDQpXaG8gX2NhcmVzXyBpZiB0aGUg
Y2xpZW50IHNlbmRzIGFuIFJQQyBjYWxsIGFmdGVyIHRoZSBsYXlvdXRyZXR1cm4/IEluDQp0aGUg
Y2FzZSBvZiBhbiB1bnJlc3BvbnNpdmUgZGF0YSBzZXJ2ZXIgdGhlIGNsaWVudCBjYW4ndCBndWFy
YW50ZWUgdGhhdA0KdGhpcyB3b24ndCBoYXBwZW4gZXZlbiBpZiBpdCBkb2VzIHdhaXQuDQpBIHBO
RlMgZmlsZXMgc2VydmVyIHRoYXQgZG9lc24ndCBwcm9wYWdhdGUgdGhlIGxheW91dHJldHVybiB0
byB0aGUgZGF0YQ0Kc2VydmVyIGluIGEgdGltZWx5IGZhc2hpb24gaXMgZnVuZGFtZW50YWxseSBf
YnJva2VuXyBpbiB0aGUgY2FzZSB3aGVyZQ0KdGhlIGNvbW11bmljYXRpb24gYmV0d2VlbiB0aGUg
ZGF0YSBzZXJ2ZXIgYW5kIGNsaWVudCBpcyBkb3duLiBJdCBjYW5ub3QNCm9mZmVyIGFueSBkYXRh
IGludGVncml0eSBndWFyYW50ZWVzIHdoZW4gdGhlIGNsaWVudCB0cmllcyB0byB3cml0ZQ0KdGhy
b3VnaCBNRFMsIGJlY2F1c2UgdGhlIERTZXMgbWF5IHN0aWxsIGJlIHByb2Nlc3Npbmcgb2xkIHdy
aXRlIFJQQw0KY2FsbHMuDQoNCj4gPj4gKExldHMgYXNzdW1lIGZpbGVzIGxheW91dCwgZm9yIGJs
b2NrcyBhbmQgb2JqZWN0cyBpdCdzIGEgYml0IGRpZmZlcmVudA0KPiA+PiAgYnV0IG1vc3RseSB0
aGUgc2FtZS4pDQo+ID4gDQo+ID4gVGhhdCwgYW5kIHRoZSBmYWN0IHRoYXQgZmVuY2luZyBoYXNu
J3QgYmVlbiBpbXBsZW1lbnRlZCBmb3IgYmxvY2tzIGFuZA0KPiA+IG9iamVjdHMuIA0KPiANCj4g
DQo+IFRoYXQncyBub3QgdHJ1ZS4gQXQgUGFuYXNhcyBhbmQgYm90aCBhdCBFTUMgdGhlcmUgaXMg
ZmVuY2luZyBpbiBwbGFjZSBhbmQNCj4gaXQgaXMgdXNlZCBldmVyeSBkYXkuIFRoaXMgaXMgd2h5
IEkgaW5zaXN0IHRoYXQgaXQgaXMgdmVyeSBtdWNoDQo+IHRoZSBzYW1lIGZvciBhbGwgb2YgdXMu
DQoNCkknbSB0YWxraW5nIGFib3V0IHRoZSB1c2Ugb2YgbGF5b3V0cmV0dXJuIGZvciBjbGllbnQg
ZmVuY2luZywgd2hpY2ggaXMNCm9ubHkgaW1wbGVtZW50ZWQgZm9yIGZpbGVzLg0KDQpIb3dldmVy
IFRhbyBhZG1pdHRlZCB0aGF0IHRoZSBibG9ja3MgY2xpZW50IGhhcyBub3QgeWV0IGltcGxlbWVu
dGVkIHRoZQ0KdGltZWQtbGVhc2UgZmVuY2luZyBhcyBkZXNjcmliZWQgaW4gUkZDNTY2Mywgc28g
dGhlcmUgaXMgc3RpbGwgd29yayB0bw0KYmUgZG9uZSB0aGVyZS4NCg0KSSd2ZSBubyBpZGVhIHdo
YXQgdGhlIG9iamVjdCBjbGllbnQgaXMgZG9pbmcuDQoNCj4gPiBUaGUgY29tbWl0IGluIHF1ZXN0
aW9uIGlzIDgyYzdjN2E1YSAoTkZTdjQuMSByZXR1cm4gdGhlIExBWU9VVA0KPiA+IGZvciBlYWNo
IGZpbGUgd2l0aCBmYWlsZWQgRFMgY29ubmVjdGlvbiBJL08pIGFuZCB0b3VjaGVzIG9ubHkNCj4g
PiBmcy9uZnMvbmZzNGZpbGVsYXlvdXQuYy4gSXQgY2Fubm90IGJlIGFmZmVjdGluZyBibG9ja3Mg
YW5kIG9iamVjdHMuDQo+ID4gDQo+IA0KPiANCj4gT0sgSSBoYWQgaW4gbWluZCB0aGUgcGF0Y2hl
cyB0aGF0IEFuZHkgc2VudC4gSSdsbCBsb29rIGFnYWluIGZvciB3aGF0DQo+IGFjdHVhbGx5IHdl
bnQgaW4uIChJdCB3YXMgYWxsIHdoaWxlIEkgd2FzIHVuYXZhaWxhYmxlKQ0KDQpIZSBzZW50IGEg
cmV2aXNlZCBwYXRjaCBzZXQsIHdoaWNoIHNob3VsZCBvbmx5IGFmZmVjdCB0aGUgZmlsZXMgbGF5
b3V0Lg0KDQo+ID4+IC0gSGVhdnkgSU8gaXMgZ29pbmcgb24sIHRoZSBkZXZpY2VfaWQgaW4gcXVl
c3Rpb24gaGFzICozKiBEU3MgaW4gaXQncw0KPiA+PiAgIGRldmljZSB0b3BvZ3JhcGh5LiBTYXkg
RFMxLCBEUzIsIERTMw0KPiA+Pg0KPiA+PiAtIFdlIGhhdmUgYmVlbiBxdWV1aW5nIElPLCBhbmQg
YWxsIHF1ZXVlcyBhcmUgZnVsbC4gKHdlIGhhdmUgMyBxdWV1ZXMgaW4NCj4gPj4gICBpbiBxdWVz
dGlvbiwgcmlnaHQ/IFdoYXQgaXMgdGhlIG1heGltdW0gUSBkZXB0aCBwZXIgZmlsZXMtRFM/IEkg
a25vdw0KPiA+PiAgIHRoYXQgaW4gYmxvY2tzIGFuZCBvYmplY3RzIHdlIHVzdWFsbHkgaGF2ZSwg
SSB0aGluaywgc29tZXRoaW5nIGxpa2UgMTI4Lg0KPiA+PiAgIFRoaXMgaXMgYSAqdHVuYWJsZSog
aW4gdGhlIGJsb2NrLWxheWVyJ3MgcmVxdWVzdC1xdWV1ZS4gSXMgaXQgbm90IHNvbWUNCj4gPj4g
ICBuZWdvdGlhdGVkIHBhcmFtZXRlciB3aXRoIHRoZSBORlMgc2VydmVycz8pDQo+ID4+DQo+ID4+
IC0gTm93LCBib29tIERTMiBoYXMgdGltZWQtb3V0LiBUaGUgTGludXgtY2xpZW50IHJlc2V0cyB0
aGUgc2Vzc2lvbiBhbmQNCj4gPj4gICBpbnRlcm5hbGx5IGNsb3NlcyBhbGwgc29ja2V0cyBvZiB0
aGF0IHNlc3Npb24uIEFsbCB0aGUgUlBDcyB0aGF0DQo+ID4+ICAgYmVsb25nIHRvIERTMiBhcmUg
YmVpbmcgcmV0dXJuZWQgdXAgd2l0aCBhIHRpbWVvdXQgZXJyb3IuIFRoaXMgb25lDQo+ID4+ICAg
aXMganVzdCB0aGUgZmlyc3Qgb2YgYWxsIHRob3NlIGJlbG9uZ2luZyB0byB0aGlzIERTMi4gVGhl
eSB3aWxsDQo+ID4+ICAgYmUgZGVjcmVtZW50aW5nIHRoZSByZWZlcmVuY2UgZm9yIHRoaXMgbGF5
b3V0IHZlcnksIHZlcnkgc29vbi4NCj4gPj4NCj4gPj4gLSBCdXQgd2hhdCBhYm91dCBEUzEsIGFu
ZCBEUzMgUlBDcy4gV2hhdCBzaG91bGQgd2UgZG8gd2l0aCB0aG9zZT8NCj4gPj4gICBUaGlzIGlz
IHdoZXJlIHlvdSBndXlzIChUcm9uZCBhbmQgQW5keSkgYXJlIHdyb25nLiBXZSBtdXN0IGFsc28N
Cj4gPj4gICB3YWl0IGZvciB0aGVzZSBSUEMncyBhcyB3ZWxsLiBBbmQgb3Bwb3NpdGUgdG8gd2hh
dCB5b3UgdGhpbmssIHRoaXMNCj4gPj4gICBzaG91bGQgbm90IHRha2UgbG9uZy4gTGV0IG1lIGV4
cGxhaW46DQo+ID4+DQo+ID4+ICAgV2UgZG9uJ3Qga25vdyBhbnl0aGluZyBhYm91dCBEUzEgYW5k
IERTMywgZWFjaCBtaWdodCBiZSwgZWl0aGVyLA0KPiA+PiAgICJIYXZpbmcgdGhlIHNhbWUgY29t
bXVuaWNhdGlvbiBwcm9ibGVtLCBsaWtlIERTMiIuIE9yICJpcyBqdXN0IHdvcmtpbmcNCj4gPj4g
ICBmaW5lIi4gU28gbGV0cyBzYXkgZm9yIGV4YW1wbGUgdGhhdCBEUzMgd2lsbCBhbHNvIHRpbWUt
b3V0IGluIHRoZQ0KPiA+PiAgIGZ1dHVyZSwgYW5kIHRoYXQgRFMxIGlzIGp1c3QgZmluZSBhbmQg
aXMgd3JpdGluZyBhcyB1c3VhbC4NCj4gPj4NCj4gPj4gICAqIERTMSAtIFNpbmNlIGl0J3Mgd29y
a2luZywgaXQgaGFzIG1vc3QgcHJvYmFibHkgYWxyZWFkeSBkb25lDQo+ID4+ICAgICB3aXRoIGFs
bCBpdCdzIElPLCBiZWNhdXNlIHRoZSBORlMgdGltZW91dCBpcyB1c3VhbGx5IG11Y2ggbG9uZ2Vy
DQo+ID4+ICAgICB0aGVuIHRoZSBub3JtYWwgUlBDIHRpbWUsIGFuZCBzaW5jZSB3ZSBhcmUgcXVl
dWluZyBldmVubHkgb24NCj4gPj4gICAgIGFsbCAzIERTcywgYXQgdGhpcyBwb2ludCBtdXN0IHBy
b2JhYmx5LCBhbGwgb2YgRFMxIFJQQ3MgYXJlDQo+ID4+ICAgICBhbHJlYWR5IGRvbmUuIChBbmQg
bGF5b3V0IGhhcyBiZWVuIGRlLXJlZmVyZW5jZWQpLg0KPiA+Pg0KPiA+PiAgICogRFMzIC0gV2ls
bCB0aW1lb3V0IGluIHRoZSBmdXR1cmUsIHdoZW4gd2lsbCB0aGF0IGJlPw0KPiA+PiAgICAgU28g
bGV0IG1lIHN0YXJ0IHdpdGgsIHNheWluZzoNCj4gPj4gICAgICgxKS4gV2UgY291bGQgZW5oYW5j
ZSBvdXIgY29kZSBhbmQgcHJvYWN0aXZlbHksIA0KPiA+PiAgICAgICAgICJjYW5jZWwvYWJvcnQi
IGFsbCBSUENzIHRoYXQgYmVsb25nIHRvIERTMyAobW9yZSBvbiB0aGlzDQo+ID4+ICAgICAgICAg
IGJlbG93KQ0KPiA+IA0KPiA+IFdoaWNoIG1ha2VzIHRoZSByYWNlIF9XT1JTRV8uIEFzIEkgc2Fp
ZCBhYm92ZSwgdGhlcmUgaXMgbm8gJ2NhbmNlbCBSUEMnDQo+ID4gb3BlcmF0aW9uIGluIFNVTlJQ
Qy4gT25jZSB5b3VyIFJQQyBjYWxsIGlzIGxhdW5jaGVkLCBpdCBjYW5ub3QgYmUNCj4gPiByZWNh
bGxlZC4gQWxsIHlvdXIgZGlzY3Vzc2lvbiBhYm92ZSBpcyBhYm91dCB0aGUgY2xpZW50IHNpZGUs
IGFuZA0KPiA+IGlnbm9yZXMgd2hhdCBtYXkgYmUgaGFwcGVuaW5nIG9uIHRoZSBkYXRhIHNlcnZl
ciBzaWRlLiBUaGUgZmVuY2luZyBpcw0KPiA+IHdoYXQgaXMgbmVlZGVkIHRvIGRlYWwgd2l0aCB0
aGUgZGF0YSBzZXJ2ZXIgcGljdHVyZS4NCj4gPiANCj4gDQo+IA0KPiBBZ2Fpbiwgc29tZSBtaXNz
IHVuZGVyc3RhbmRpbmcuIEkgbmV2ZXIgc2FpZCB3ZSBzaG91bGQgbm90IHNlbmQNCj4gYSBMQVlP
VVRfUkVUVVJOIGJlZm9yZSB3cml0aW5nIHRocm91Z2ggTURTLiBUaGUgb3Bwb3NpdGUgaXMgdHJ1
ZSwNCj4gSSB0aGluayBpdCBpcyBhIG5vdmVsIGlkZWEgYW5kIGdpdmVzIHlvdSB0aGUga2luZCBv
ZiBiYXJyaWVyIHRoYXQNCj4gd2lsbCBoYXJkZW4gYW5kIHJvYnVzdCB0aGUgc3lzdGVtLg0KPiAN
Cj4gICAgV0hBVCBJJ20gc2F5aW5nIGlzIHRoYXQgdGhpcyBjYW5ub3QgaGFwcGVuIHdoaWxlIHRo
ZSBzY2hpem9waHJlbmljDQo+ICAgIGNsaWVudCBpcyBidXNpbHkgc3RpbGwgYWN0aXZlbHkgc2ti
LXNlbmRpbmcgbW9yZSBhbmQgbW9yZSBieXRlcw0KPiAgICB0byBhbGwgdGhlIG90aGVyIERTcyBp
biB0aGUgbGF5b3V0LiBMT05HIEFGVEVSIFRIRSBMQVlPVVRfUkVUVVJODQo+ICAgIEhBUyBCRUVO
IFNFTlQgQU5EIFJFU1BPTkRFRC4NCj4gDQo+IFNvIHdoYXQgeW91IGFyZSBzYXlpbmcgZG9lcyBu
b3QgYXQgYWxsIGNvbnRyYWRpY3RzIHdoYXQgSSB3YW50Lg0KPiANCj4gICAgIlRoZSBmZW5jaW5n
IGlzIHdoYXQgaXMgbmVlZGVkIHRvIGRlYWwgd2l0aCB0aGUgZGF0YSBzZXJ2ZXIgcGljdHVyZSIN
Cj4gICAgIA0KPiAgICAgRmluZSBCdXQgT05MWSBhZnRlciB0aGUgY2xpZW50IGhhcyByZWFsbHkg
c3RvcHBlZCBhbGwgc2VuZHMuDQo+ICAgICAoRWFjaCBvbmUgd2lsbCBkbyBpdCdzIGpvYikNCj4g
DQo+IEJUVzogVGhlIFNlcnZlciBkb2VzIG5vdCAqbmVlZCogdGhlIENsaWVudCB0byBzZW5kIGEg
TEFZT1VUX1JFVFVSTg0KPiAgICAgIEl0J3MganVzdCBhIG5pY2UtdG8taGF2ZSwgd2hpY2ggSSdt
IGZpbmUgd2l0aC4NCj4gICAgICBCb3RoIFBhbmFzYXMgYW5kIEVNQyB3aGVuIElPIGlzIHNlbnQg
dGhyb3VnaCBNRFMsIHdpbGwgZmlyc3QNCj4gICAgICByZWNhbGwsIG92ZXJsYXBwaW5nIGxheW91
dHMsIGFuZCBvbmx5IHRoZW4gcHJvY2VlZCB3aXRoDQo+ICAgICAgTURTIHByb2Nlc3NpbmcuIChU
aGlzIGlzIHNvbWUgZGVlcGx5IHJvb3RlZCBtZWNoYW5pc20gaW5zaWRlDQo+ICAgICAgdGhlIEZT
LCBhbiBNRFMgYmVpbmcganVzdCBhbm90aGVyIGNsaWVudCkNCg0KQWdhaW4sIHdlJ3JlIG5vdCB0
YWxraW5nIGFib3V0IGJsb2NrcyBvciBvYmplY3RzLg0KYQ0KPiAgICAgIFNvIHRoaXMgaXMgYSBr
bm93biBwcm9ibGVtIHRoYXQgaXMgdGFrZW4gY2FyZSBvZi4gQnV0IEkgdG90YWxseQ0KPiAgICAg
IGFncmVlIHdpdGggeW91LCB0aGUgY2xpZW50IExBWU9VVF9SRVRVUk4oaW5nKSB0aGUgbGF5b3V0
IHdpbGwgc2F2ZQ0KPiAgICAgIGxvdHMgb2YgcHJvdG9jb2wgdGltZSBieSBhdm9pZGluZyB0aGUg
cmVjYWxscy4NCj4gICAgICBOb3cgeW91IHVuZGVyc3RhbmQgd2h5IGluIE9iamVjdHMgd2UgbWFu
ZGF0ZWQgdGhpcyBMQVlPVVRfUkVUVVJODQo+ICAgICAgb24gZXJyb3JzLiBBbmQgd2hpbGUgYXQg
aXQgd2Ugd2FudCB0aGUgZXhhY3QgZXJyb3IgcmVwb3J0ZWQuDQo+IA0KPiA+PiAgICAgKDIpLiBP
ciBXZSBjYW4gcHJvdmUgdGhhdCBEUzMncyBSUENzIHdpbGwgdGltZW91dCBhdCB3b3JzdA0KPiA+
PiAgICAgICAgICBjYXNlIDEgeCBORlMtdGltZW91dCBhZnRlciBhYm92ZSBEUzIgdGltZW91dCBl
dmVudCwgb3INCj4gPj4gICAgICAgICAgMiB4IE5GUy10aW1lb3V0IGFmdGVyIHRoZSBxdWV1aW5n
IG9mIHRoZSBmaXJzdCB0aW1lZC1vdXQNCj4gPj4gICAgICAgICAgUlBDLiBBbmQgc3RhdGlzdGlj
YWxseSBpbiB0aGUgYXZlcmFnZSBjYXNlIERTMyB3aWxsIHRpbWVvdXQNCj4gPj4gICAgICAgICAg
dmVyeSBuZWFyIHRoZSB0aW1lIERTMiB0aW1lZC1vdXQuDQo+ID4+DQo+ID4+ICAgICAgICAgIFRo
aXMgaXMgZWFzeSBzaW5jZSB0aGUgbGFzdCBJTyB3ZSBxdWV1ZWQgd2FzIHRoZSBvbmUgdGhhdA0K
PiA+PiAgICAgICAgICBtYWRlIERTMidzIHF1ZXVlIHRvIGJlIGZ1bGwsIGFuZCBpdCB3YXMga2Vw
dCBmdWxsIGJlY2F1c2UNCj4gPj4gICAgICAgICAgRFMyIHN0b3BwZWQgcmVzcG9uZGluZyBhbmQg
bm90aGluZyBlbXB0aWVkIHRoZSBxdWV1ZS4NCj4gPj4NCj4gPj4gICAgICBTbyB0aGUgZWFzaWVz
dCB3ZSBjYW4gZG8gaXMgd2FpdCBmb3IgRFMzIHRvIHRpbWVvdXQsIHNvb24NCj4gPj4gICAgICBl
bm91Z2gsIGFuZCBvbmNlIHRoYXQgd2lsbCBoYXBwZW4sIHNlc3Npb24gd2lsbCBiZSByZXNldCBh
bmQgYWxsDQo+ID4+ICAgICAgUlBDcyB3aWxsIGVuZCB3aXRoIGFuIGVycm9yLg0KPiA+IA0KPiA+
IA0KPiA+IFlvdSBhcmUgc3RpbGwgb25seSBkaXNjdXNzaW5nIHRoZSBjbGllbnQgc2lkZS4NCj4g
PiANCj4gPiBSZWFkIG15IGxpcHM6IFN1biBSUEMgT1BFUkFUSU9OUyBETyBOT1QgVElNRU9VVCBB
TkQgQ0FOTk9UIEJFIEFCT1JURUQgT1INCj4gPiBDQU5DRUxFRC4gRmVuY2luZyBpcyB0aGUgY2xv
c2VzdCB3ZSBjYW4gY29tZSB0byBhbiBhYm9ydCBvcGVyYXRpb24uDQo+ID4gDQo+IA0KPiANCj4g
QWdhaW4gSSBkaWQgbm90IG1lYW4gdGhlICJTdW4gUlBDIE9QRVJBVElPTlMiIG9uIHRoZSB3aXJl
LiBJIG1lYW50DQo+IHRoZSBMaW51eC1yZXF1ZXN0LWVudGl0eSB3aGljaCB3aGlsZSBleGlzdCBo
YXMgYSBwb3RlbnRpYWwgdG8gYmUNCj4gc3VibWl0dGVkIGZvciBza2Itc2VuZC4gQXMgc2VlbiBh
Ym92ZSB0aGVzZSBlbnRpdGllcyBkbyB0aW1lb3V0DQo+IGluICItbyBzb2Z0IiBtb2RlIGFuZCBv
bmNlIHJlbGVhc2VkIHJlbW92ZSB0aGUgcG90ZW50aWFsIG9mIGFueSBtb3JlDQo+IGZ1dHVyZSBz
a2Itc2VuZHMgb24gdGhlIHdpcmUuDQo+IA0KPiBCVVQgd2hhdCBJIGRvIG5vdCB1bmRlcnN0YW5k
IGlzOiBJbiBhYm92ZSBleGFtcGxlIHdlIGFyZSB0YWxraW5nDQo+IGFib3V0IERTMy4gV2UgYXNz
dW1lZCB0aGF0IERTMyBoYXMgYSBjb21tdW5pY2F0aW9uIHByb2JsZW0uIFNvIG5vIGFtb3VudA0K
PiBvZiAiZmVuY2luZyIgb3IgdnVkdSBvciBhbnkgb3RoZXIga2luZCBvZiBvcGVyYXRpb24gY2Fu
IGV2ZXIgYWZmZWN0DQo+IHRoZSBjbGllbnQgcmVnYXJkaW5nIERTMy4gQmVjYXVzZSBldmVuIGlm
IE9uLXRoZS1zZXJ2ZXIgcGVuZGluZyByZXF1ZXN0cw0KPiBmcm9tIGNsaWVudCBvbiBEUzMgYXJl
IGZlbmNlZCBhbmQgZGlzY2FyZGVkIHRoZXNlIGVycm9ycyB3aWxsIG5vdA0KPiBiZSBjb21tdW5p
Y2F0ZWQgYmFjayB0aGUgY2xpZW50LiBUaGUgY2xpZW50IHdpbGwgc2l0IGlkbGUgb24gRFMzDQo+
IGNvbW11bmljYXRpb24gdW50aWwgdGhlIGVuZCBvZiB0aGUgdGltZW91dCwgcmVnYXJkbGVzcy4N
Cg0KV2UgZG9uJ3QgY2FyZSBhYm91dCBhbnkgcmVjZWl2aW5nIHRoZSBlcnJvcnMuIFdlJ3ZlIHRp
bWVkIG91dC4gQWxsIHdlDQp3YW50IHRvIGRvIGlzIHRvIGZlbmNlIG9mZiB0aGUgZGFtbmVkIHdy
aXRlcyB0aGF0IGhhdmUgYWxyZWFkeSBiZWVuIHNlbnQNCnRvIHRoZSBib3JrZW4gRFMgYW5kIHRo
ZW4gcmVzZW5kIHRoZW0gdGhyb3VnaCB0aGUgTURTLg0KDQo+IEFjdHVhbGx5IHdoYXQgSSBwcm9w
b3NlIGZvciBEUzMgaW4gdGhlIGJlc3Qgcm9idXN0IGNsaWVudCBpcyB0bw0KPiBkZXN0cm95IGl0
J3MgRFMzJ3Mgc2Vzc2lvbnMgYW5kIHRoZXJlZm9yIGNhdXNlIGFsbCBMaW51eC1yZXF1ZXN0LWVu
dGl0aWVzDQo+IHRvIHJldHVybiBtdWNoIG11Y2ggZmFzdGVyLCB0aGVuIGlmICpqdXN0IHdhaXRp
bmcqIGZvciB0aGUgdGltZW91dCB0byBleHBpcmUuDQoNCkkgcmVwZWF0OiBkZXN0cm95aW5nIHRo
ZSBzZXNzaW9uIG9uIHRoZSBjbGllbnQgZG9lcyBOT1RISU5HIHRvIGhlbHAgeW91Lg0KDQo+ID4+
IFNvIGluIHRoZSB3b3JzdCBjYXNlIHNjZW5hcmlvIHdlIGNhbiByZWNvdmVyIDIgeCBORlMtdGlt
ZW91dCBhZnRlcg0KPiA+PiBhIG5ldHdvcmsgcGFydGl0aW9uLCB3aGljaCBpcyBqdXN0IDEgeCBO
RlMtdGltZW91dCwgYWZ0ZXIgeW91cg0KPiA+PiBzY2hpem9waHJlbmljIEZFTkNFX01FX09GRiwg
bmV3bHkgaW52ZW50ZWQgb3BlcmF0aW9uLg0KPiA+Pg0KPiA+PiBXaGF0IHdlIGNhbiBkbyB0byBl
bmhhbmNlIG91ciBjb2RlIHRvIHJlZHVjZSBlcnJvciByZWNvdmVyeSB0bw0KPiA+PiAxIHggTkZT
LXRpbWVvdXQ6DQo+ID4+DQo+ID4+IC0gRFMzIGFib3ZlOg0KPiA+PiAgIChBcyBJIHNhaWQgRFMx
J3MgcXVldWVzIGFyZSBub3cgZW1wdHksIGJlY2F1c2UgaXQgd2FzIHdvcmtpbmcgZmluZSwNCj4g
Pj4gICAgU28gRFMzIGlzIGEgcmVwcmVzZW50YXRpb24gb2YgYWxsIERTJ3MgdGhhdCBoYXZlIFJQ
Q3MgYXQgdGhlDQo+ID4+ICAgIHRpbWUgRFMyIHRpbWVkLW91dCwgd2hpY2ggYmVsb25nIHRvIHRo
aXMgbGF5b3V0KQ0KPiA+Pg0KPiA+PiAgIFdlIGNhbiBwcm9hY3RpdmVseSBhYm9ydCBhbGwgUlBD
cyBiZWxvbmdpbmcgdG8gRFMzLiBJZiB0aGVyZSBpcw0KPiA+PiAgIGEgd2F5IHRvIGludGVybmFs
bHkgYWJvcnQgUlBDJ3MgdXNlIHRoYXQuIEVsc2UganVzdCByZXNldCBpdCdzDQo+ID4+ICAgc2Vz
c2lvbiBhbmQgYWxsIHNvY2tldHMgd2lsbCBjbG9zZSAoYW5kIHJlb3BlbiksIGFuZCBhbGwgUlBD
J3MNCj4gPj4gICB3aWxsIGVuZCB3aXRoIGEgZGlzY29ubmVjdCBlcnJvci4NCj4gPiANCj4gPiBO
b3Qgb24gbW9zdCBzZXJ2ZXJzIHRoYXQgSSdtIGF3YXJlIG9mLiBJZiB5b3UgY2xvc2Ugb3IgcmVz
ZXQgdGhlIHNvY2tldA0KPiA+IG9uIHRoZSBjbGllbnQsIHRoZW4gdGhlIExpbnV4IHNlcnZlciB3
aWxsIGhhcHBpbHkgY29udGludWUgdG8gcHJvY2Vzcw0KPiA+IHRob3NlIFJQQyBjYWxsczsgaXQg
anVzdCB3b24ndCBiZSBhYmxlIHRvIHNlbmQgYSByZXBseS4NCj4gPiBGdXJ0aGVybW9yZSwgaWYg
dGhlIHByb2JsZW0gaXMgdGhhdCB0aGUgZGF0YSBzZXJ2ZXIgaXNuJ3QgcmVzcG9uZGluZywNCj4g
PiB0aGVuIGEgc29ja2V0IGNsb3NlL3Jlc2V0IHRlbGxzIHlvdSBub3RoaW5nIGVpdGhlci4NCj4g
PiANCj4gDQo+IA0KPiBBZ2FpbiBJJ20gdGFsa2luZyBhYm91dCB0aGUgTkZTLUludGVybmFsLXJl
cXVlc3QtZW50aXRpZXMgdGhlc2Ugd2lsbA0KPiBiZSByZWxlYXNlZCwgdGhvdWdoIGd1YXJhbnR5
aW5nIHRoYXQgbm8gbW9yZSB0aHJlYWRzIHdpbGwgdXNlIGFueSBvZg0KPiB0aGVzZSB0byBzZW5k
IGFueSBtb3JlIGJ5dGVzIG92ZXIgdG8gYW55IERTcy4NCj4gDQo+IEFORCB5ZXMsIHllcy4gT25j
ZSB0aGUgY2xpZW50IGhhcyBkb25lIGl0J3Mgam9iIGFuZCBzdG9wcGVkIGFueSBmdXR1cmUNCj4g
c2tiLXNlbmRzIHRvICphbGwqIERTcyBpbiBxdWVzdGlvbiwgb25seSB0aGVuIGl0IGNhbiByZXBv
cnQgdG8gTURTOg0KPiAgICJIZXkgSSdtIGRvbmUgc2VuZGluZyBpbiBhbGwgb3RoZXIgcm91dHMg
aGVyZSBMQVlPVVRfUkVUVVJOIg0KPiAgIChOb3cgZmVuY2luZyBoYXBwZW5zIG9uIHNlcnZlcnMp
DQo+ICAgDQo+ICAgYW5kIGNsaWVudCBnb2VzIG9uIGFuZCBzYXlzDQo+ICAgDQo+ICAgIkhleSBj
YW4geW91IE1EUyBwbGVhc2UgYWxzbyB3cml0ZSB0aGlzIGRhdGEiDQo+IA0KPiBXaGljaCBpcyBw
ZXJmZWN0IGZvciBNRFMgYmVjYXVzZSBvdGhlcndpc2UgaWYgaXQgd2FudHMgdG8gbWFrZSBzdXJl
LA0KPiBpdCB3aWxsIG5lZWQgdG8gcmVjYWxsIGFsbCBvdXRzdGFuZGluZyBsYXlvdXRzLCBleGFj
dGx5IGZvciB5b3VyDQo+IHJlYXNvbiwgZm9yIGNvbmNlcm4gZm9yIHRoZSBkYXRhIGNvcnJ1cHRp
b24gdGhhdCBjYW4gaGFwcGVuLg0KDQpTbyBpdCByZWNhbGxzIHRoZSBsYXlvdXRzLCBhbmQgdGhl
bi4uLiBpdCBfc3RpbGxfIGhhcyB0byBmZW5jZSBvZmYgYW55DQp3cml0ZXMgdGhhdCBhcmUgaW4g
cHJvZ3Jlc3Mgb24gdGhlIGJyb2tlbiBEUy4NCg0KQWxsIHlvdSd2ZSBkb25lIGlzIGFkZCBhIHJl
Y2FsbCB0byB0aGUgd2hvbGUgcHJvY2Vzcy4gV2h5Pw0KDQo+ID4+IC0gQm90aCBEUzIgdGhhdCB0
aW1lZC1vdXQsIGFuZCBEUzMgdGhhdCB3YXMgYWJvcnRlZC4gU2hvdWxkIGJlDQo+ID4+ICAgbWFy
a2VkIHdpdGggYSBmbGFnLiBXaGVuIG5ldyBJTyB0aGF0IGJlbG9uZyB0byBzb21lIG90aGVyDQo+
ID4+ICAgaW5vZGUgdGhyb3VnaCBzb21lIG90aGVyIGxheW91dCtkZXZpY2VfaWQgZW5jb3VudGVy
cyBhIGZsYWdnZWQNCj4gPj4gICBkZXZpY2UsIGl0IHNob3VsZCBhYm9ydCBhbmQgdHVybiB0byBN
RFMgSU8sIHdpdGggYWxzbyBpbnZhbGlkYXRpbmcNCj4gPj4gICBpdCdzIGxheW91dCwgYW5kIGhl
bnMsIHNvb24gZW5vdWdoIHRoZSBkZXZpY2VfaWQgZm9yIERTMiYzIHdpbGwgYmUNCj4gPj4gICBk
ZS1yZWZlcmVuY2VkIGFuZCBiZSByZW1vdmVkIGZyb20gZGV2aWNlIGNhY2hlLiAoQW5kIGFsbCBy
ZWZlcmVuY2luZw0KPiA+PiAgIGxheW91dHMgYXJlIG5vdyBnb25lKQ0KPiA+IA0KPiA+IFRoZXJl
IGlzIG5vIFJQQyBhYm9ydCBmdW5jdGlvbmFsaXR5IFN1biBSUEMuIEFnYWluLCB0aGlzIGFyZ3Vt
ZW50IHJlbGllcw0KPiA+IG9uIGZ1bmN0aW9uYWxpdHkgdGhhdCBfZG9lc24ndF8gZXhpc3QuDQo+
ID4gDQo+IA0KPiANCj4gQWdhaW4gSSBtZWFuIGludGVybmFsbHkgYXQgdGhlIGNsaWVudC4gRm9y
IGV4YW1wbGUgY2xvc2luZyB0aGUgc29ja2V0IHdpbGwNCj4gaGF2ZSB0aGUgZWZmZWN0IEkgd2Fu
dC4gKEFuZCBzb21lIG90aGVyIHRyaWNrcyB3ZSBjYW4gdGFsayBhYm91dCB0aG9zZQ0KPiBsYXRl
ciwgbGV0cyBhZ3JlZSBhYm91dCB0aGUgcHJpbmNpcGFsIGZpcnN0KQ0KDQpUaW1pbmcgb3V0IHdp
bGwgcHJldmVudCB0aGUgZGFtbmVkIGNsaWVudCBmcm9tIHNlbmRpbmcgbW9yZSBkYXRhLiBTbw0K
d2hhdD8NCg0KPiA+PiAgIFNvIHdlIGRvIG5vdCBjb250aW51ZSBxdWV1aW5nIG5ldyBJTyB0byBk
ZWFkIGRldmljZXMuIEFuZCBzaW5jZSBtb3N0DQo+ID4+ICAgcHJvYmFibHkgTURTIHdpbGwgbm90
IGdpdmUgdXMgZGVhZCBzZXJ2ZXJzIGluIG5ldyBsYXlvdXQsIHdlIHNob3VsZCBiZQ0KPiA+PiAg
IGdvb2QuDQo+ID4+IEluIHN1bW1lcnkuDQo+ID4+IC0gRkVOQ0VfTUVfT0ZGIGlzIGEgbmV3IG9w
ZXJhdGlvbiwgYW5kIGlzIG5vdCA9PT0gTEFZT1VUX1JFVFVSTi4gQ2xpZW50DQo+ID4+ICAgKm11
c3Qgbm90KiBza2Itc2VuZCBhIHNpbmdsZSBieXRlIGJlbG9uZ2luZyB0byBhIGxheW91dCwgYWZ0
ZXIgdGhlIHNlbmQNCj4gPj4gICBvZiBMQVlPVVRfUkVUVVJOLg0KPiA+PiAgIChJdCBuZWVkIG5v
dCB3YWl0IGZvciBPUFRfRE9ORSBmcm9tIERTIHRvIGRvIHRoYXQsIGl0IGp1c3QgbXVzdCBtYWtl
DQo+ID4+ICAgIHN1cmUsIHRoYXQgYWxsIGl0J3MgaW50ZXJuYWwsIG9yIG9uLXRoZS13aXJlIHJl
cXVlc3QsIGFyZSBhYm9ydGVkDQo+ID4+ICAgIGJ5IGVhc2lseSBjbG9zaW5nIHRoZSBzb2NrZXRz
IHRoZXkgYmVsb25nIHRvbywgYW5kL29yIHdhaXRpbmcgZm9yDQo+ID4+ICAgIGhlYWx0aHkgRFMn
cyBJTyB0byBiZSBPUFRfRE9ORSAuIFNvIHRoZSBjbGllbnQgaXMgbm90IGRlcGVuZGVudCBvbg0K
PiA+PiAgICBhbnkgRFMgcmVzcG9uc2UsIGl0IGlzIG9ubHkgZGVwZW5kZW50IG9uIGl0J3MgaW50
ZXJuYWwgc3RhdGUgYmVpbmcNCj4gPj4gICAgKmNsZWFuKiBmcm9tIGFueSBtb3JlIHNrYi1zZW5k
KHMpKQ0KPiA+IA0KPiA+IERpdHRvDQo+ID4gDQo+ID4+IC0gVGhlIHByb3BlciBpbXBsZW1lbnRh
dGlvbiBvZiBMQVlPVVRfUkVUVVJOIG9uIGVycm9yIGZvciBmYXN0IHR1cm5vdmVyDQo+ID4+ICAg
aXMgbm90IGhhcmQsIGFuZCBkb2VzIG5vdCBpbnZvbHZlIGEgbmV3IGludmVudGVkIE5GUyBvcGVy
YXRpb24gc3VjaA0KPiA+PiAgIGFzIEZFTkNFX01FX09GRi4gUHJvcGVyIGNvZGRlZCBjbGllbnQs
IGluZGVwZW5kZW50bHksIHdpdGhvdXQNCj4gPj4gICB0aGUgYWlkIG9mIGFueSBGRU5DRV9NRV9P
RkYgb3BlcmF0aW9uLCBjYW4gYWNoaWV2ZSBhIGZhc3RlciB0dXJuYXJvdW5kDQo+ID4+ICAgYnkg
YWN0aXZlbHkgcmV0dXJuaW5nIGFsbCBsYXlvdXRzIHRoYXQgYmVsb25nIHRvIGEgYmFkIERTLCBh
bmQgbm90DQo+ID4+ICAgd2FpdGluZyBmb3IgYSBmZW5jZS1vZmYgb2YgYSBzaW5nbGUgbGF5b3V0
LCB0aGVuIGVuY291bnRlcmluZyBqdXN0DQo+ID4+ICAgdGhlIHNhbWUgZXJyb3Igd2l0aCBhbGwg
b3RoZXIgbGF5b3V0cyB0aGF0IGhhdmUgdGhlIHNhbWUgRFMgICAgIA0KPiA+IA0KPiA+IFdoYXQg
ZG8geW91IG1lYW4gYnkgImFsbCBsYXlvdXRzIHRoYXQgYmVsb25nIHRvIGEgYmFkIERTIj8gTGF5
b3V0cyBkb24ndA0KPiA+IGJlbG9uZyB0byBhIERTLCBhbmQgc28gdGhlcmUgaXMgbm8gd2F5IHRv
IGdldCBmcm9tIGEgRFMgdG8gYSBsYXlvdXQuDQo+ID4gDQo+IA0KPiANCj4gV2h5LCBzdXJlLiBs
b29wIG9uIGFsbCBsYXlvdXRzIGFuZCBhc2sgaWYgaXQgaGFzIGEgc3BlY2lmaWMgRFMuDQoNCg0K
DQo+ID4+IC0gQW5kIEkga25vdyB0aGF0IGp1c3QgYXMgeW91IGRpZCBub3QgcmVhZCBteSBlbWFp
bHMgZnJvbSBiZWZvcmUNCj4gPj4gICBtZSBnb2luZyB0byBIb3NwaXRhbCwgeW91IHdpbGwgY29u
dGludWUgdG8gbm90IHVuZGVyc3RhbmQgdGhpcw0KPiA+PiAgIG9uZSwgb3Igd2hhdCBJJ20gdHJ5
aW5nIHRvIGV4cGxhaW4sIGFuZCB3aWxsIG1vc3QgcHJvYmFibHkgaWdub3JlDQo+ID4+ICAgYWxs
IG9mIGl0LiBCdXQgcGxlYXNlIG5vdGUgb25lIHRoaW5nOg0KPiA+IA0KPiA+IEkgcmVhZCB0aGVt
LCBidXQganVzdCBhcyBub3csIHRoZXkgY29udGludWUgdG8gaWdub3JlIHRoZSByZWFsaXR5IGFi
b3V0DQo+ID4gdGltZW91dHM6IHRpbWVvdXRzIG1lYW4gX25vdGhpbmdfIGluIGFuIFJQQyBmYWls
b3ZlciBzaXR1YXRpb24uIFRoZXJlIGlzDQo+ID4gbm8gUlBDIGFib3J0IGZ1bmN0aW9uYWxpdHkg
dGhhdCB5b3UgY2FuIHJlbHkgb24gb3RoZXIgdGhhbiBmZW5jaW5nLg0KPiA+IA0KPiANCj4gDQo+
IEkgaG9wZSBJIGV4cGxhaW5lZCB0aGlzIGJ5IG5vdy4gSWYgbm90IHBsZWFzZSwgcGxlYXNlLCBs
ZXRzIG9yZ2FuaXplDQo+IGEgcGhvbmUgY2FsbC4gV2UgY2FuIHVzZSBQYW5hc2FzIGNvbmZlcmVu
Y2UgbnVtYmVyIHdoZW5ldmVyIHlvdSBhcmUNCj4gYXZhaWxhYmxlLiBJIHRoaW5rIHdlIGNvbW11
bmljYXRlIGJldHRlciBpbiBwZXJzb24uDQo+IA0KPiBFdmVyeW9uZSBlbHNlIGlzIGFsc28gaW52
aXRlZC4NCj4gDQo+IEJVVCB0aGVyZSBpcyBvbmUgbW9zdCBpbXBvcnRhbnQgcG9pbnQgZm9yIG1l
Og0KPiANCj4gICAgQXMgc3RhdGVkIGJ5IHRoZSBSRkMuIENsaWVudCBtdXN0IGd1YXJhbnR5IHRo
YXQgbm8gbW9yZSBieXRlcyB3aWxsIGJlDQo+ICAgIHNlbnQgdG8gYW55IERTcyBpbiBhIGxheW91
dCwgb25jZSBMQVlPVVRfUkVUVVJOIGlzIHNlbnQuIFRoaXMgaXMgdGhlDQo+ICAgIG9ubHkgZGVm
aW5pdGlvbiBvZiBMQVlPVVRfUkVUVVJOLCBhbmQgTk9fTUFUQ0hJTkdfTEFZT1VUIGFzIHJlc3Bv
bnNlDQo+ICAgIHRvIGEgTEFZT1VUX1JFQ0FMTC4gV2hpY2ggaXM6DQo+ICAgIENsaWVudCBoYXMg
aW5kaWNhdGVkIG5vIG1vcmUgZnV0dXJlIHNlbmRzIG9uIGEgbGF5b3V0LiAoQW5kIHNlcnZlciB3
aWxsDQo+ICAgIGVuZm9yY2UgaXQgd2l0aCBhIGZlbmNpbmcpDQoNClRoZSBjbGllbnQgY2FuJ3Qg
Z3VhcmFudGVlIHRoYXQuIFRoZSBwcm90b2NvbCBvZmZlcnMgbm8gd2F5IGZvciBpdCB0byBkbw0K
c28sIG5vIG1hdHRlciB3aGF0IHRoZSBwTkZTIHRleHQgbWF5IGNob29zZSB0byBzYXkuDQoNCj4g
Pj4gICAgIFlPVSBoYXZlIHNhYm90YWdlZCB0aGUgTkZTIDQuMSBMaW51eCBjbGllbnQsIHdoaWNo
IGlzIG5vdyB0b3RhbGx5DQo+ID4+ICAgICBub3QgU1REIGNvbXBsYWludCwgYW5kIGhhdmUgaW50
cm9kdWNlZCBDUkFTSHMuIEFuZCBmb3Igbm8gZ29vZA0KPiA+PiAgICAgcmVhc29uLg0KPiA+IA0K
PiA+IFNlZSBhYm92ZS4NCj4gPiANCj4gDQo+IA0KPiBPSyBXZSdsbCBoYXZlIHRvIHNlZSBhYm91
dCB0aGVzZSBjcmFzaGVzLCBsZXRzIHRhbGsgYWJvdXQgdGhlbS4NCj4gDQo+IFRoYW5rcw0KPiBC
b2F6DQoNCi0tIA0KVHJvbmQgTXlrbGVidXN0DQpMaW51eCBORlMgY2xpZW50IG1haW50YWluZXIN
Cg0KTmV0QXBwDQpUcm9uZC5NeWtsZWJ1c3RAbmV0YXBwLmNvbQ0Kd3d3Lm5ldGFwcC5jb20NCg0K
T24gVHVlLCAyMDEyLTA4LTE0IGF0IDAzOjI4ICswMzAwLCBCb2F6IEhhcnJvc2ggd3JvdGU6DQo+
IE9uIDA4LzE0LzIwMTIgMDM6MTYgQU0sIE15a2xlYnVzdCwgVHJvbmQgd3JvdGU6DQo+IA0KPiA+
IA0KPiA+IFRoZSBjbGllbnQgY2FuJ3QgZ3VhcmFudGVlIHRoYXQuIFRoZSBwcm90b2NvbCBvZmZl
cnMgbm8gd2F5IGZvciBpdCB0byBkbw0KPiA+IHNvLCBubyBtYXR0ZXIgd2hhdCB0aGUgcE5GUyB0
ZXh0IG1heSBjaG9vc2UgdG8gc2F5Lg0KPiA+IA0KPiANCj4gDQo+IFdoYXQ/IHdoeSBub3Q/IGFs
bCB0aGUgY2xpZW50IG5lZWRzIHRvIGRvIGlzIHN0b3Agc2VuZGluZyBieXRlcy4NCj4gV2hhdCBp
cyBub3QgZ3VhcmFudGllZD8gSSBjb21wbGV0ZWx5IGRvIG5vdCB1bmRlcnN0YW5kIHdoYXQgeW91
DQo+IHNheS4gSG93IHN0b3BwaW5nIGFueSBzZW5kcyBkb2VzIG5vdCBndWFyYW50eSB0aGF0Pw0K
DQpJZiB0aGUgY2xpZW50IGhhcyBsb3N0IGNvbnRyb2wgb2YgdGhlIHRyYW5zcG9ydCwgdGhlbiBp
dCBoYXMgbm8gY29udHJvbA0Kb3ZlciB3aGF0IHRoZSBkYXRhIHNlcnZlciBzZWVzIGFuZCB3aGVu
LiBJdCBjYW4gcHJvY2VzcyB0aGUgY2xpZW50J3MNCndyaXRlIFJQQyA1IG1pbnV0ZXMgZnJvbSBu
b3csIGFuZCBpdCB3aWxsIG5ldmVyIGtub3cuDQoNClRIQVQgaXMgd2hhdCBpcyBhYnN1cmQgYWJv
dXQgdGhlIHdob2xlICJ0aGUgY2xpZW50IE1VU1QgTk9UIHNlbmQuLi4iDQpzaGViYW5nLg0KVEhB
VCBpcyB3aHkgdGhlcmUgaXMgbm8gZ3VhcmFudGVlLg0KDQotLSANClRyb25kIE15a2xlYnVzdA0K
TGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFwcA0KVHJvbmQuTXlrbGVidXN0QG5l
dGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQoNCg==
On Thu, Aug 9, 2012 at 11:39 PM, Myklebust, Trond
<[email protected]> wrote:
> On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>> <[email protected]> wrote:
>> > On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>> >> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>> >> <[email protected]> wrote:
>> >> > Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>> >> > disconnected data server) we've been sending layoutreturn calls
>> >> > while there is potentially still outstanding I/O to the data
>> >> > servers. The reason we do this is to avoid races between replayed
>> >> > writes to the MDS and the original writes to the DS.
>> >> >
>> >> > When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>> >> > be triggered because it assumes that we would never call
>> >> > layoutreturn without knowing that all I/O to the DS is
>> >> > finished. The fix is to remove the BUG_ON() now that the
>> >> > assumptions behind the test are obsolete.
>> >> >
>> >> Isn't MDS supposed to recall the layout if races are possible between
>> >> outstanding write-to-DS and write-through-MDS?
>> >
>> > Where do you read that in RFC5661?
>> >
>> That's my (maybe mis-)understanding of how server works... But looking
>> at rfc5661 section 18.44.3. layoutreturn implementation.
>> "
>> After this call,
>> the client MUST NOT use the returned layout(s) and the associated
>> storage protocol to access the file data.
>> "
>> And given commit 0a57cdac3f, client is using the layout even after
>> layoutreturn, which IMHO is a violation of rfc5661.
>
> No. It is using the layoutreturn to tell the MDS to fence off I/O to a
> data server that is not responding. It isn't attempting to use the
> layout after the layoutreturn: the whole point is that we are attempting
> write-through-MDS after the attempt to write through the DS timed out.
>
But it is RFC violation that there is in-flight DS IO when client
sends layoutreturn, right? Not just in-flight, client is well possible
to send IO to DS _after_ layoutreturn because some thread can hold
lseg reference and not yet send IO.
>> >> And it causes data corruption for blocklayout if client returns layout
>> >> while there is in-flight disk IO...
>> >
>> > Then it needs to turn off fast failover to write-through-MDS.
>> >
>> If you still consider it following rfc5661, I'd choose to disable
>> layoutreturn in before write-through-MDS for blocklayout, by adding
>> some flag like PNFS_NO_LAYOUTRET_ON_FALLTHRU similar to objects'
>> PNFS_LAYOUTRET_ON_SETATTR.
>
> I don't see how that will prevent corruption.
>
> In fact, IIRC, in NFSv4.2 Sorin's proposed changes specifically use the
> layoutreturn to communicate to the MDS that the DS is timing out via an
> error code (like the object layout has done all the time). How can you
> reconcile that change with a flag such as the one you propose?
I just intend to use the flag to disable layoutreturn in
pnfs_ld_write_done. block extents are data access permissions per
rfc5663. When we don't layoutreturn in pnfs_ld_write_done(), block
layout works correctly because server can decide if there is data
access race and if there is, MDS can recall the layout from client
before applying the MDS writes.
Sorin's proposed error code is just a client indication to server that
there is disk access error. It is not intended to solve the data race
between write-through-MDS and write-through-DS.
Thanks,
Tao
On 08/09/2012 06:39 PM, Myklebust, Trond wrote:
> If the problem is that the DS is failing to respond, how does the client
> know that the in-flight I/O has ended?
For the client, the above DS in question, has timed-out, we have reset
it's session and closed it's sockets. And all it's RPC requests have
been, or are being, ended with a timeout-error. So the timed-out
DS is a no-op. All it's IO request will end very soon, if not already.
A DS time-out is just a very valid, and meaningful response, just like
an op-done-with-error. This was what Andy added to the RFC's errata
which I agree with.
>
> No. It is using the layoutreturn to tell the MDS to fence off I/O to a
> data server that is not responding. It isn't attempting to use the
> layout after the layoutreturn:
> the whole point is that we are attempting
> write-through-MDS after the attempt to write through the DS timed out.
>
Trond STOP!!! this is pure bullshit. You guys took the opportunity of
me being in Hospital, and the rest of the bunch not having a clue. And
snuck in a patch that is totally wrong for everyone, not taking care of
any other LD *crashes* . And especially when this patch is wrong even for
files layout.
This above here is where you are wrong!! You don't understand my point,
and ignore my comments. So let me state it as clear as I can.
(Lets assume files layout, for blocks and objects it's a bit different
but mostly the same.)
- Heavy IO is going on, the device_id in question has *3* DSs in it's
device topography. Say DS1, DS2, DS3
- We have been queuing IO, and all queues are full. (we have 3 queues in
in question, right? What is the maximum Q depth per files-DS? I know
that in blocks and objects we usually have, I think, something like 128.
This is a *tunable* in the block-layer's request-queue. Is it not some
negotiated parameter with the NFS servers?)
- Now, boom DS2 has timed-out. The Linux-client resets the session and
internally closes all sockets of that session. All the RPCs that
belong to DS2 are being returned up with a timeout error. This one
is just the first of all those belonging to this DS2. They will
be decrementing the reference for this layout very, very soon.
- But what about DS1, and DS3 RPCs. What should we do with those?
This is where you guys (Trond and Andy) are wrong. We must also
wait for these RPC's as well. And opposite to what you think, this
should not take long. Let me explain:
We don't know anything about DS1 and DS3, each might be, either,
"Having the same communication problem, like DS2". Or "is just working
fine". So lets say for example that DS3 will also time-out in the
future, and that DS1 is just fine and is writing as usual.
* DS1 - Since it's working, it has most probably already done
with all it's IO, because the NFS timeout is usually much longer
then the normal RPC time, and since we are queuing evenly on
all 3 DSs, at this point must probably, all of DS1 RPCs are
already done. (And layout has been de-referenced).
* DS3 - Will timeout in the future, when will that be?
So let me start with, saying:
(1). We could enhance our code and proactively,
"cancel/abort" all RPCs that belong to DS3 (more on this
below)
(2). Or We can prove that DS3's RPCs will timeout at worst
case 1 x NFS-timeout after above DS2 timeout event, or
2 x NFS-timeout after the queuing of the first timed-out
RPC. And statistically in the average case DS3 will timeout
very near the time DS2 timed-out.
This is easy since the last IO we queued was the one that
made DS2's queue to be full, and it was kept full because
DS2 stopped responding and nothing emptied the queue.
So the easiest we can do is wait for DS3 to timeout, soon
enough, and once that will happen, session will be reset and all
RPCs will end with an error.
So in the worst case scenario we can recover 2 x NFS-timeout after
a network partition, which is just 1 x NFS-timeout, after your
schizophrenic FENCE_ME_OFF, newly invented operation.
What we can do to enhance our code to reduce error recovery to
1 x NFS-timeout:
- DS3 above:
(As I said DS1's queues are now empty, because it was working fine,
So DS3 is a representation of all DS's that have RPCs at the
time DS2 timed-out, which belong to this layout)
We can proactively abort all RPCs belonging to DS3. If there is
a way to internally abort RPC's use that. Else just reset it's
session and all sockets will close (and reopen), and all RPC's
will end with a disconnect error.
- Both DS2 that timed-out, and DS3 that was aborted. Should be
marked with a flag. When new IO that belong to some other
inode through some other layout+device_id encounters a flagged
device, it should abort and turn to MDS IO, with also invalidating
it's layout, and hens, soon enough the device_id for DS2&3 will be
de-referenced and be removed from device cache. (And all referencing
layouts are now gone)
So we do not continue queuing new IO to dead devices. And since most
probably MDS will not give us dead servers in new layout, we should be
good.
In summery.
- FENCE_ME_OFF is a new operation, and is not === LAYOUT_RETURN. Client
*must not* skb-send a single byte belonging to a layout, after the send
of LAYOUT_RETURN.
(It need not wait for OPT_DONE from DS to do that, it just must make
sure, that all it's internal, or on-the-wire request, are aborted
by easily closing the sockets they belong too, and/or waiting for
healthy DS's IO to be OPT_DONE . So the client is not dependent on
any DS response, it is only dependent on it's internal state being
*clean* from any more skb-send(s))
- The proper implementation of LAYOUT_RETURN on error for fast turnover
is not hard, and does not involve a new invented NFS operation such
as FENCE_ME_OFF. Proper codded client, independently, without
the aid of any FENCE_ME_OFF operation, can achieve a faster turnaround
by actively returning all layouts that belong to a bad DS, and not
waiting for a fence-off of a single layout, then encountering just
the same error with all other layouts that have the same DS
- And I know that just as you did not read my emails from before
me going to Hospital, you will continue to not understand this
one, or what I'm trying to explain, and will most probably ignore
all of it. But please note one thing:
YOU have sabotaged the NFS 4.1 Linux client, which is now totally
not STD complaint, and have introduced CRASHs. And for no good
reason.
No thanks
Boaz
On Fri, Aug 10, 2012 at 12:29 AM, Myklebust, Trond
<[email protected]> wrote:
> On Fri, 2012-08-10 at 00:22 +0800, Peng Tao wrote:
>> On Thu, Aug 9, 2012 at 11:39 PM, Myklebust, Trond
>> <[email protected]> wrote:
>> > On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>> >> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>> >> <[email protected]> wrote:
>> >> > On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>> >> >> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>> >> >> <[email protected]> wrote:
>> >> >> > Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>> >> >> > disconnected data server) we've been sending layoutreturn calls
>> >> >> > while there is potentially still outstanding I/O to the data
>> >> >> > servers. The reason we do this is to avoid races between replayed
>> >> >> > writes to the MDS and the original writes to the DS.
>> >> >> >
>> >> >> > When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>> >> >> > be triggered because it assumes that we would never call
>> >> >> > layoutreturn without knowing that all I/O to the DS is
>> >> >> > finished. The fix is to remove the BUG_ON() now that the
>> >> >> > assumptions behind the test are obsolete.
>> >> >> >
>> >> >> Isn't MDS supposed to recall the layout if races are possible between
>> >> >> outstanding write-to-DS and write-through-MDS?
>> >> >
>> >> > Where do you read that in RFC5661?
>> >> >
>> >> That's my (maybe mis-)understanding of how server works... But looking
>> >> at rfc5661 section 18.44.3. layoutreturn implementation.
>> >> "
>> >> After this call,
>> >> the client MUST NOT use the returned layout(s) and the associated
>> >> storage protocol to access the file data.
>> >> "
>> >> And given commit 0a57cdac3f, client is using the layout even after
>> >> layoutreturn, which IMHO is a violation of rfc5661.
>> >
>> > No. It is using the layoutreturn to tell the MDS to fence off I/O to a
>> > data server that is not responding. It isn't attempting to use the
>> > layout after the layoutreturn: the whole point is that we are attempting
>> > write-through-MDS after the attempt to write through the DS timed out.
>> >
>> But it is RFC violation that there is in-flight DS IO when client
>> sends layoutreturn, right? Not just in-flight, client is well possible
>> to send IO to DS _after_ layoutreturn because some thread can hold
>> lseg reference and not yet send IO.
>
> Once the write has been sent, how do you know that it is no longer
> 'in-flight' unless the DS responds?
>
>> >> >> And it causes data corruption for blocklayout if client returns layout
>> >> >> while there is in-flight disk IO...
>> >> >
>> >> > Then it needs to turn off fast failover to write-through-MDS.
>> >> >
>> >> If you still consider it following rfc5661, I'd choose to disable
>> >> layoutreturn in before write-through-MDS for blocklayout, by adding
>> >> some flag like PNFS_NO_LAYOUTRET_ON_FALLTHRU similar to objects'
>> >> PNFS_LAYOUTRET_ON_SETATTR.
>> >
>> > I don't see how that will prevent corruption.
>> >
>> > In fact, IIRC, in NFSv4.2 Sorin's proposed changes specifically use the
>> > layoutreturn to communicate to the MDS that the DS is timing out via an
>> > error code (like the object layout has done all the time). How can you
>> > reconcile that change with a flag such as the one you propose?
>> I just intend to use the flag to disable layoutreturn in
>> pnfs_ld_write_done. block extents are data access permissions per
>> rfc5663. When we don't layoutreturn in pnfs_ld_write_done(), block
>> layout works correctly because server can decide if there is data
>> access race and if there is, MDS can recall the layout from client
>> before applying the MDS writes.
>>
>> Sorin's proposed error code is just a client indication to server that
>> there is disk access error. It is not intended to solve the data race
>> between write-through-MDS and write-through-DS.
>
> Then how do you solve that race on a block device?
As mentioned above, block extents are permissions per RFC5663. So if
MDS needs to access the disk, it needs the permission as well. So if
there is data access race, MDS must recall the layout from client
before processing the MDS writes. We've been dealing with the problem
for years in MPFS and it works perfectly to rely on MDS's decisions.
Thanks,
Tao
On Tue, Aug 14, 2012 at 9:45 PM, Myklebust, Trond
<[email protected]> wrote:
> On Tue, 2012-08-14 at 10:48 +0300, Benny Halevy wrote:
>> On 2012-08-09 18:39, Myklebust, Trond wrote:
>> > On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>> >> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>> >> <[email protected]> wrote:
>> >>> On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>> >>>> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>> >>>> <[email protected]> wrote:
>> >>>>> Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>> >>>>> disconnected data server) we've been sending layoutreturn calls
>> >>>>> while there is potentially still outstanding I/O to the data
>> >>>>> servers. The reason we do this is to avoid races between replayed
>> >>>>> writes to the MDS and the original writes to the DS.
>> >>>>>
>> >>>>> When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>> >>>>> be triggered because it assumes that we would never call
>> >>>>> layoutreturn without knowing that all I/O to the DS is
>> >>>>> finished. The fix is to remove the BUG_ON() now that the
>> >>>>> assumptions behind the test are obsolete.
>> >>>>>
>> >>>> Isn't MDS supposed to recall the layout if races are possible between
>> >>>> outstanding write-to-DS and write-through-MDS?
>> >>>
>> >>> Where do you read that in RFC5661?
>> >>>
>> >> That's my (maybe mis-)understanding of how server works... But looking
>> >> at rfc5661 section 18.44.3. layoutreturn implementation.
>> >> "
>> >> After this call,
>> >> the client MUST NOT use the returned layout(s) and the associated
>> >> storage protocol to access the file data.
>> >> "
>> >> And given commit 0a57cdac3f, client is using the layout even after
>> >> layoutreturn, which IMHO is a violation of rfc5661.
>> >
>> > No. It is using the layoutreturn to tell the MDS to fence off I/O to a
>> > data server that is not responding. It isn't attempting to use the
>> > layout after the layoutreturn: the whole point is that we are attempting
>> > write-through-MDS after the attempt to write through the DS timed out.
>> >
>>
>> I hear you, but this use case is valid after a time out / disconnect
>> (which will translate to PNFS_OSD_ERR_UNREACHABLE for the objects layout)
>> In other cases, I/Os to the DS might obviously be in flight and the BUG_ON
>> indicates that.
>>
>> IMO, the right way to implement that is to initially mark the lsegs invalid
>> and increment plh_block_lgets, as we do today in _pnfs_return_layout
>> but actually send the layout return only when the last segment is dereferenced.
>
> This is what we do for object and block layout types, so your
> objects-specific objection is unfounded.
>
object layout is also doing layout return on IO error (commit
fe0fe83585f8). And it doesn't take care of draining concurrent
in-flight IO. I guess that's why Boaz saw the same BUG_ON.
--
Thanks,
Tao
On Fri, Aug 10, 2012 at 12:29 AM, Myklebust, Trond
<[email protected]> wrote:
> On Fri, 2012-08-10 at 00:22 +0800, Peng Tao wrote:
>> On Thu, Aug 9, 2012 at 11:39 PM, Myklebust, Trond
>> <[email protected]> wrote:
>> > On Thu, 2012-08-09 at 23:01 +0800, Peng Tao wrote:
>> >> On Thu, Aug 9, 2012 at 10:36 PM, Myklebust, Trond
>> >> <[email protected]> wrote:
>> >> > On Thu, 2012-08-09 at 22:30 +0800, Peng Tao wrote:
>> >> >> On Thu, Aug 9, 2012 at 4:21 AM, Trond Myklebust
>> >> >> <[email protected]> wrote:
>> >> >> > Ever since commit 0a57cdac3f (NFSv4.1 send layoutreturn to fence
>> >> >> > disconnected data server) we've been sending layoutreturn calls
>> >> >> > while there is potentially still outstanding I/O to the data
>> >> >> > servers. The reason we do this is to avoid races between replayed
>> >> >> > writes to the MDS and the original writes to the DS.
>> >> >> >
>> >> >> > When this happens, the BUG_ON() in nfs4_layoutreturn_done can
>> >> >> > be triggered because it assumes that we would never call
>> >> >> > layoutreturn without knowing that all I/O to the DS is
>> >> >> > finished. The fix is to remove the BUG_ON() now that the
>> >> >> > assumptions behind the test are obsolete.
>> >> >> >
>> >> >> Isn't MDS supposed to recall the layout if races are possible between
>> >> >> outstanding write-to-DS and write-through-MDS?
>> >> >
>> >> > Where do you read that in RFC5661?
>> >> >
>> >> That's my (maybe mis-)understanding of how server works... But looking
>> >> at rfc5661 section 18.44.3. layoutreturn implementation.
>> >> "
>> >> After this call,
>> >> the client MUST NOT use the returned layout(s) and the associated
>> >> storage protocol to access the file data.
>> >> "
>> >> And given commit 0a57cdac3f, client is using the layout even after
>> >> layoutreturn, which IMHO is a violation of rfc5661.
>> >
>> > No. It is using the layoutreturn to tell the MDS to fence off I/O to a
>> > data server that is not responding. It isn't attempting to use the
>> > layout after the layoutreturn: the whole point is that we are attempting
>> > write-through-MDS after the attempt to write through the DS timed out.
>> >
>> But it is RFC violation that there is in-flight DS IO when client
>> sends layoutreturn, right? Not just in-flight, client is well possible
>> to send IO to DS _after_ layoutreturn because some thread can hold
>> lseg reference and not yet send IO.
>
> Once the write has been sent, how do you know that it is no longer
> 'in-flight' unless the DS responds?
RFC5663 provides a way.
"
"blh_maximum_io_time" is the maximum
time it can take for a client I/O to the storage system to either
complete or fail
"
It is not perfect solution but still serves as a best effort. It
solves the in-flight IO question for current writing thread.
For in-flight IO from other concurrent threads, lseg reference is the
source that we can rely on. And I think that the BUG_ON can be
triggered much easily because of concurrent writing threads and one of
them fails DS writes.
--
Thanks,
Tao
On 08/14/2012 03:16 AM, Myklebust, Trond wrote:
>
> The client can't guarantee that. The protocol offers no way for it to do
> so, no matter what the pNFS text may choose to say.
>
What? why not? all the client needs to do is stop sending bytes.
What is not guarantied? I completely do not understand what you
say. How stopping any sends does not guaranty that?
Please explain
Thanks
Boaz
T24gVHVlLCAyMDEyLTA4LTE0IGF0IDIyOjMwICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gT24g
VHVlLCBBdWcgMTQsIDIwMTIgYXQgOTo0NSBQTSwgTXlrbGVidXN0LCBUcm9uZA0KPiA8VHJvbmQu
TXlrbGVidXN0QG5ldGFwcC5jb20+IHdyb3RlOg0KPiA+IE9uIFR1ZSwgMjAxMi0wOC0xNCBhdCAx
MDo0OCArMDMwMCwgQmVubnkgSGFsZXZ5IHdyb3RlOg0KPiA+PiBPbiAyMDEyLTA4LTA5IDE4OjM5
LCBNeWtsZWJ1c3QsIFRyb25kIHdyb3RlOg0KPiA+PiA+IE9uIFRodSwgMjAxMi0wOC0wOSBhdCAy
MzowMSArMDgwMCwgUGVuZyBUYW8gd3JvdGU6DQo+ID4+ID4+IE9uIFRodSwgQXVnIDksIDIwMTIg
YXQgMTA6MzYgUE0sIE15a2xlYnVzdCwgVHJvbmQNCj4gPj4gPj4gPFRyb25kLk15a2xlYnVzdEBu
ZXRhcHAuY29tPiB3cm90ZToNCj4gPj4gPj4+IE9uIFRodSwgMjAxMi0wOC0wOSBhdCAyMjozMCAr
MDgwMCwgUGVuZyBUYW8gd3JvdGU6DQo+ID4+ID4+Pj4gT24gVGh1LCBBdWcgOSwgMjAxMiBhdCA0
OjIxIEFNLCBUcm9uZCBNeWtsZWJ1c3QNCj4gPj4gPj4+PiA8VHJvbmQuTXlrbGVidXN0QG5ldGFw
cC5jb20+IHdyb3RlOg0KPiA+PiA+Pj4+PiBFdmVyIHNpbmNlIGNvbW1pdCAwYTU3Y2RhYzNmIChO
RlN2NC4xIHNlbmQgbGF5b3V0cmV0dXJuIHRvIGZlbmNlDQo+ID4+ID4+Pj4+IGRpc2Nvbm5lY3Rl
ZCBkYXRhIHNlcnZlcikgd2UndmUgYmVlbiBzZW5kaW5nIGxheW91dHJldHVybiBjYWxscw0KPiA+
PiA+Pj4+PiB3aGlsZSB0aGVyZSBpcyBwb3RlbnRpYWxseSBzdGlsbCBvdXRzdGFuZGluZyBJL08g
dG8gdGhlIGRhdGENCj4gPj4gPj4+Pj4gc2VydmVycy4gVGhlIHJlYXNvbiB3ZSBkbyB0aGlzIGlz
IHRvIGF2b2lkIHJhY2VzIGJldHdlZW4gcmVwbGF5ZWQNCj4gPj4gPj4+Pj4gd3JpdGVzIHRvIHRo
ZSBNRFMgYW5kIHRoZSBvcmlnaW5hbCB3cml0ZXMgdG8gdGhlIERTLg0KPiA+PiA+Pj4+Pg0KPiA+
PiA+Pj4+PiBXaGVuIHRoaXMgaGFwcGVucywgdGhlIEJVR19PTigpIGluIG5mczRfbGF5b3V0cmV0
dXJuX2RvbmUgY2FuDQo+ID4+ID4+Pj4+IGJlIHRyaWdnZXJlZCBiZWNhdXNlIGl0IGFzc3VtZXMg
dGhhdCB3ZSB3b3VsZCBuZXZlciBjYWxsDQo+ID4+ID4+Pj4+IGxheW91dHJldHVybiB3aXRob3V0
IGtub3dpbmcgdGhhdCBhbGwgSS9PIHRvIHRoZSBEUyBpcw0KPiA+PiA+Pj4+PiBmaW5pc2hlZC4g
VGhlIGZpeCBpcyB0byByZW1vdmUgdGhlIEJVR19PTigpIG5vdyB0aGF0IHRoZQ0KPiA+PiA+Pj4+
PiBhc3N1bXB0aW9ucyBiZWhpbmQgdGhlIHRlc3QgYXJlIG9ic29sZXRlLg0KPiA+PiA+Pj4+Pg0K
PiA+PiA+Pj4+IElzbid0IE1EUyBzdXBwb3NlZCB0byByZWNhbGwgdGhlIGxheW91dCBpZiByYWNl
cyBhcmUgcG9zc2libGUgYmV0d2Vlbg0KPiA+PiA+Pj4+IG91dHN0YW5kaW5nIHdyaXRlLXRvLURT
IGFuZCB3cml0ZS10aHJvdWdoLU1EUz8NCj4gPj4gPj4+DQo+ID4+ID4+PiBXaGVyZSBkbyB5b3Ug
cmVhZCB0aGF0IGluIFJGQzU2NjE/DQo+ID4+ID4+Pg0KPiA+PiA+PiBUaGF0J3MgbXkgKG1heWJl
IG1pcy0pdW5kZXJzdGFuZGluZyBvZiBob3cgc2VydmVyIHdvcmtzLi4uIEJ1dCBsb29raW5nDQo+
ID4+ID4+IGF0IHJmYzU2NjEgc2VjdGlvbiAxOC40NC4zLiBsYXlvdXRyZXR1cm4gaW1wbGVtZW50
YXRpb24uDQo+ID4+ID4+ICINCj4gPj4gPj4gQWZ0ZXIgdGhpcyBjYWxsLA0KPiA+PiA+PiAgICB0
aGUgY2xpZW50IE1VU1QgTk9UIHVzZSB0aGUgcmV0dXJuZWQgbGF5b3V0KHMpIGFuZCB0aGUgYXNz
b2NpYXRlZA0KPiA+PiA+PiAgICBzdG9yYWdlIHByb3RvY29sIHRvIGFjY2VzcyB0aGUgZmlsZSBk
YXRhLg0KPiA+PiA+PiAiDQo+ID4+ID4+IEFuZCBnaXZlbiBjb21taXQgMGE1N2NkYWMzZiwgY2xp
ZW50IGlzIHVzaW5nIHRoZSBsYXlvdXQgZXZlbiBhZnRlcg0KPiA+PiA+PiBsYXlvdXRyZXR1cm4s
IHdoaWNoIElNSE8gaXMgYSB2aW9sYXRpb24gb2YgcmZjNTY2MS4NCj4gPj4gPg0KPiA+PiA+IE5v
LiBJdCBpcyB1c2luZyB0aGUgbGF5b3V0cmV0dXJuIHRvIHRlbGwgdGhlIE1EUyB0byBmZW5jZSBv
ZmYgSS9PIHRvIGENCj4gPj4gPiBkYXRhIHNlcnZlciB0aGF0IGlzIG5vdCByZXNwb25kaW5nLiBJ
dCBpc24ndCBhdHRlbXB0aW5nIHRvIHVzZSB0aGUNCj4gPj4gPiBsYXlvdXQgYWZ0ZXIgdGhlIGxh
eW91dHJldHVybjogdGhlIHdob2xlIHBvaW50IGlzIHRoYXQgd2UgYXJlIGF0dGVtcHRpbmcNCj4g
Pj4gPiB3cml0ZS10aHJvdWdoLU1EUyBhZnRlciB0aGUgYXR0ZW1wdCB0byB3cml0ZSB0aHJvdWdo
IHRoZSBEUyB0aW1lZCBvdXQuDQo+ID4+ID4NCj4gPj4NCj4gPj4gSSBoZWFyIHlvdSwgYnV0IHRo
aXMgdXNlIGNhc2UgaXMgdmFsaWQgYWZ0ZXIgYSB0aW1lIG91dCAvIGRpc2Nvbm5lY3QNCj4gPj4g
KHdoaWNoIHdpbGwgdHJhbnNsYXRlIHRvIFBORlNfT1NEX0VSUl9VTlJFQUNIQUJMRSBmb3IgdGhl
IG9iamVjdHMgbGF5b3V0KQ0KPiA+PiBJbiBvdGhlciBjYXNlcywgSS9PcyB0byB0aGUgRFMgbWln
aHQgb2J2aW91c2x5IGJlIGluIGZsaWdodCBhbmQgdGhlIEJVR19PTg0KPiA+PiBpbmRpY2F0ZXMg
dGhhdC4NCj4gPj4NCj4gPj4gSU1PLCB0aGUgcmlnaHQgd2F5IHRvIGltcGxlbWVudCB0aGF0IGlz
IHRvIGluaXRpYWxseSBtYXJrIHRoZSBsc2VncyBpbnZhbGlkDQo+ID4+IGFuZCBpbmNyZW1lbnQg
cGxoX2Jsb2NrX2xnZXRzLCBhcyB3ZSBkbyB0b2RheSBpbiBfcG5mc19yZXR1cm5fbGF5b3V0DQo+
ID4+IGJ1dCBhY3R1YWxseSBzZW5kIHRoZSBsYXlvdXQgcmV0dXJuIG9ubHkgd2hlbiB0aGUgbGFz
dCBzZWdtZW50IGlzIGRlcmVmZXJlbmNlZC4NCj4gPg0KPiA+IFRoaXMgaXMgd2hhdCB3ZSBkbyBm
b3Igb2JqZWN0IGFuZCBibG9jayBsYXlvdXQgdHlwZXMsIHNvIHlvdXINCj4gPiBvYmplY3RzLXNw
ZWNpZmljIG9iamVjdGlvbiBpcyB1bmZvdW5kZWQuDQo+ID4NCj4gb2JqZWN0IGxheW91dCBpcyBh
bHNvIGRvaW5nIGxheW91dCByZXR1cm4gb24gSU8gZXJyb3IgKGNvbW1pdA0KPiBmZTBmZTgzNTg1
ZjgpLiBBbmQgaXQgZG9lc24ndCB0YWtlIGNhcmUgb2YgZHJhaW5pbmcgY29uY3VycmVudA0KPiBp
bi1mbGlnaHQgSU8uIEkgZ3Vlc3MgdGhhdCdzIHdoeSBCb2F6IHNhdyB0aGUgc2FtZSBCVUdfT04u
DQoNClllcy4gSSBkaWQgbm90aWNlIHRoYXQgY29kZSB3aGVuIEkgd2FzIGxvb2tpbmcgaW50byB0
aGlzLiBIb3dldmVyIHRoYXQncw0KQm9heidzIG93biBwYXRjaCwgYW5kIGl0IF9vbmx5XyBhcHBs
aWVzIHRvIHRoZSBvYmplY3RzIGxheW91dCB0eXBlLiBJDQphc3N1bWVkIHRoYXQgaGUgaGFkIHRl
c3RlZCBpdCB3aGVuIEkgYXBwbGllZCBpdC4uLg0KDQpPbmUgd2F5IHRvIGZpeCB0aGF0IHdvdWxk
IGJlIHRvIGtlZXAgYSBjb3VudCBvZiAib3V0c3RhbmRpbmcNCnJlYWQvd3JpdGVzIiBpbiB0aGUg
bGF5b3V0LCBzbyB0aGF0IHdoZW4gdGhlIGVycm9yIG9jY3VycywgYW5kIHdlIHdhbnQNCnRvIGZh
bGwgYmFjayB0byBNRFMsIHdlIGp1c3QgaW5jcmVtZW50IHBsaF9ibG9ja19sZ2V0cywgaW52YWxp
ZGF0ZSB0aGUNCmxheW91dCwgYW5kIHRoZW4gbGV0IHRoZSBvdXRzdGFuZGluZyByZWFkL3dyaXRl
cyBmYWxsIHRvIHplcm8gYmVmb3JlDQpzZW5kaW5nIHRoZSBsYXlvdXRyZXR1cm4uDQpJZiB0aGUg
b2JqZWN0cyBsYXlvdXQgd2FudHMgdG8gZG8gdGhhdCwgdGhlbiBJIGhhdmUgbm8gb2JqZWN0aW9u
LiBBcw0KSSd2ZSBzYWlkIG11bHRpcGxlIHRpbWVzLCB0aG91Z2gsIEknbSBub3QgY29udmluY2Vk
IHdlIHdhbnQgdG8gZG8gdGhhdA0KZm9yIHRoZSBmaWxlcyBsYXlvdXQuDQoNCi0tIA0KVHJvbmQg
TXlrbGVidXN0DQpMaW51eCBORlMgY2xpZW50IG1haW50YWluZXINCg0KTmV0QXBwDQpUcm9uZC5N
eWtsZWJ1c3RAbmV0YXBwLmNvbQ0Kd3d3Lm5ldGFwcC5jb20NCg0K
T24gVGh1LCAyMDEyLTA4LTA5IGF0IDIyOjMwICswODAwLCBQZW5nIFRhbyB3cm90ZToNCj4gT24g
VGh1LCBBdWcgOSwgMjAxMiBhdCA0OjIxIEFNLCBUcm9uZCBNeWtsZWJ1c3QNCj4gPFRyb25kLk15
a2xlYnVzdEBuZXRhcHAuY29tPiB3cm90ZToNCj4gPiBFdmVyIHNpbmNlIGNvbW1pdCAwYTU3Y2Rh
YzNmIChORlN2NC4xIHNlbmQgbGF5b3V0cmV0dXJuIHRvIGZlbmNlDQo+ID4gZGlzY29ubmVjdGVk
IGRhdGEgc2VydmVyKSB3ZSd2ZSBiZWVuIHNlbmRpbmcgbGF5b3V0cmV0dXJuIGNhbGxzDQo+ID4g
d2hpbGUgdGhlcmUgaXMgcG90ZW50aWFsbHkgc3RpbGwgb3V0c3RhbmRpbmcgSS9PIHRvIHRoZSBk
YXRhDQo+ID4gc2VydmVycy4gVGhlIHJlYXNvbiB3ZSBkbyB0aGlzIGlzIHRvIGF2b2lkIHJhY2Vz
IGJldHdlZW4gcmVwbGF5ZWQNCj4gPiB3cml0ZXMgdG8gdGhlIE1EUyBhbmQgdGhlIG9yaWdpbmFs
IHdyaXRlcyB0byB0aGUgRFMuDQo+ID4NCj4gPiBXaGVuIHRoaXMgaGFwcGVucywgdGhlIEJVR19P
TigpIGluIG5mczRfbGF5b3V0cmV0dXJuX2RvbmUgY2FuDQo+ID4gYmUgdHJpZ2dlcmVkIGJlY2F1
c2UgaXQgYXNzdW1lcyB0aGF0IHdlIHdvdWxkIG5ldmVyIGNhbGwNCj4gPiBsYXlvdXRyZXR1cm4g
d2l0aG91dCBrbm93aW5nIHRoYXQgYWxsIEkvTyB0byB0aGUgRFMgaXMNCj4gPiBmaW5pc2hlZC4g
VGhlIGZpeCBpcyB0byByZW1vdmUgdGhlIEJVR19PTigpIG5vdyB0aGF0IHRoZQ0KPiA+IGFzc3Vt
cHRpb25zIGJlaGluZCB0aGUgdGVzdCBhcmUgb2Jzb2xldGUuDQo+ID4NCj4gSXNuJ3QgTURTIHN1
cHBvc2VkIHRvIHJlY2FsbCB0aGUgbGF5b3V0IGlmIHJhY2VzIGFyZSBwb3NzaWJsZSBiZXR3ZWVu
DQo+IG91dHN0YW5kaW5nIHdyaXRlLXRvLURTIGFuZCB3cml0ZS10aHJvdWdoLU1EUz8NCg0KV2hl
cmUgZG8geW91IHJlYWQgdGhhdCBpbiBSRkM1NjYxPw0KDQo+IEFuZCBpdCBjYXVzZXMgZGF0YSBj
b3JydXB0aW9uIGZvciBibG9ja2xheW91dCBpZiBjbGllbnQgcmV0dXJucyBsYXlvdXQNCj4gd2hp
bGUgdGhlcmUgaXMgaW4tZmxpZ2h0IGRpc2sgSU8uLi4NCg0KVGhlbiBpdCBuZWVkcyB0byB0dXJu
IG9mZiBmYXN0IGZhaWxvdmVyIHRvIHdyaXRlLXRocm91Z2gtTURTLg0KDQotLSANClRyb25kIE15
a2xlYnVzdA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFwcA0KVHJvbmQuTXlr
bGVidXN0QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQoNCg==