Hello!
I've reported this a few times now, but we still keep hitting the issue.
NFS clients in the cluster occasionally get stuck with the socket to one
of the NFS servers in CLOSE_WAIT state, and don't recover.
The NFS servers (knfsd) used to be running 2.6.30 since newer kernels hit
XFS bugs, but are all now finally running 3.2.10 or 3.2.16.
I've added some debugging printk()s and WARN()s to the kernel to try to
figure out what is going on. Without knowing much about how it's actually
expected to work :), these were mostly to figure out why the socket is
being closed. I added the following on top of with Trond's sunrpc
debugging patch:
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 1e3b42c..c850c76 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1462,12 +1462,16 @@ call_status(struct rpc_task *task)
rpc_delay(task, 3*HZ);
case -ETIMEDOUT:
task->tk_action = call_timeout;
+ printk("%s: RPC call timeout (%d), disconnecting\n",
+ clnt->cl_protname, -status);
if (task->tk_client->cl_discrtry)
xprt_conditional_disconnect(task->tk_xprt,
req->rq_connect_cookie);
break;
case -ECONNRESET:
case -ECONNREFUSED:
+ printk("%s: RPC connection reset (%d), rebinding\n",
+ clnt->cl_protname, -status);
rpc_force_rebind(clnt);
rpc_delay(task, 3*HZ);
case -EPIPE:
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index f1c57d4..9dc7870 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -633,7 +633,12 @@ void xprt_force_disconnect(struct rpc_xprt *xprt)
{
/* Don't race with the test_bit() in xprt_clear_locked() */
spin_lock_bh(&xprt->transport_lock);
+#if 0
set_bit(XPRT_CLOSE_WAIT, &xprt->state);
+#else
+ if (test_and_set_bit(XPRT_CLOSE_WAIT, &xprt->state) == 0)
+ WARN(1, "xprt_force_disconnect(): setting XPRT_CLOSE_WAIT\n");
+#endif
/* Try to schedule an autoclose RPC call */
if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
BUG_ON(!queue_work(rpciod_workqueue, &xprt->task_cleanup));
@@ -660,7 +665,12 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
goto out;
if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
goto out;
+#if 0
set_bit(XPRT_CLOSE_WAIT, &xprt->state);
+#else
+ if (test_and_set_bit(XPRT_CLOSE_WAIT, &xprt->state) == 0)
+ WARN(1, "xprt_conditional_disconnect(): setting XPRT_CLOSE_WAIT\n");
+#endif
/* Try to schedule an autoclose RPC call */
if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
BUG_ON(!queue_work(rpciod_workqueue, &xprt->task_cleanup));
@@ -706,8 +716,10 @@ void xprt_connect(struct rpc_task *task)
if (!xprt_lock_write(xprt, task))
return;
- if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
+ if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
xprt->ops->close(xprt);
+ printk("RPC: xprt_connect: cleared XPRT_CLOSE_WAIT for xprt %p (pid %u)\n", xprt, task->tk_pid);
+ }
if (xprt_connected(xprt))
xprt_release_write(xprt, task);
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 55472c4..0ed3b99 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -772,6 +772,8 @@ static void xs_tcp_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
if (req->rq_bytes_sent == req->rq_snd_buf.len)
goto out_release;
set_bit(XPRT_CLOSE_WAIT, &task->tk_xprt->state);
+ WARN(1, "xs_tcp_release_xprt(): setting XPRT_CLOSE_WAIT -- req->rq_bytes_sent=%u, req->rq_snd_buf.len=%u\n",
+ req->rq_bytes_sent, req->rq_snd_buf.len);
out_release:
xprt_release_xprt(xprt, task);
}
@@ -1517,6 +1519,11 @@ static void xs_tcp_state_change(struct sock *sk)
/* The server initiated a shutdown of the socket */
xprt_force_disconnect(xprt);
xprt->connect_cookie++;
+ printk("RPC: server initated shutdown -- state %x conn %d dead %d zapped %d sk_shutdown %d\n",
+ sk->sk_state, xprt_connected(xprt),
+ sock_flag(sk, SOCK_DEAD),
+ sock_flag(sk, SOCK_ZAPPED),
+ sk->sk_shutdown);
case TCP_CLOSING:
/*
* If the server closed down the connection, make sure that
---
Mount options on the clients: rw,hard,intr,tcp,timeo=300,retrans=2,vers=3
...the timeout is primarily left over from an attempt to make HA
fail-over "reconnection" times faster, since with UDP it actually did
make a significant difference. Does this even matter with TCP sockets?
Here, we hit the CLOSE_WAIT state. There are a number of occasional
"RPC call timeout (110), disconnecting" in normal operation; I'm not sure
why this is.
[1119196.640017] nfs: RPC call timeout (110), disconnecting
[1119196.720026] nfs: RPC call timeout (110), disconnecting
[1119196.892021] nfs: RPC call timeout (110), disconnecting
[1119198.516022] nfs: RPC call timeout (110), disconnecting
[1119220.576025] nfs: RPC call timeout (110), disconnecting
[1119871.048040] nfs: RPC call timeout (110), disconnecting
[1120266.464021] nfs: RPC call timeout (110), disconnecting
[1120341.472021] nfs: RPC call timeout (110), disconnecting
[1122139.008022] nfs: RPC call timeout (110), disconnecting
[1122166.112043] nfs: RPC call timeout (110), disconnecting
[1122302.304037] nfs: RPC call timeout (110), disconnecting
[1122666.735765] ------------[ cut here ]------------
[1122666.735855] WARNING: at net/sunrpc/xprt.c:640 xprt_force_disconnect+0x4a/0xa0()
[1122666.735945] Hardware name: PowerEdge 1950
[1122666.736025] xprt_force_disconnect(): setting XPRT_CLOSE_WAIT
[1122666.736099] Modules linked in: ipmi_devintf ipmi_si ipmi_msghandler xt_recent nf_conntrack_ftp xt_state xt_owner nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 bnx2 serio_raw evdev
[1122666.736898] Pid: 18458, comm: httpd Tainted: G W 3.2.10-hw+ #16
[1122666.736975] Call Trace:
[1122666.737043] <IRQ> [<ffffffff816b545a>] ? xprt_force_disconnect+0x4a/0xa0
[1122666.737176] [<ffffffff8105dd60>] warn_slowpath_common+0x80/0xc0
[1122666.737252] [<ffffffff8105de99>] warn_slowpath_fmt+0x69/0x70
[1122666.737335] [<ffffffffa0055aff>] ? tcp_packet+0x7cf/0x10f0 [nf_conntrack]
[1122666.737414] [<ffffffff81702c41>] ? _raw_spin_lock_bh+0x11/0x40
[1122666.737490] [<ffffffff816b545a>] xprt_force_disconnect+0x4a/0xa0
[1122666.737566] [<ffffffff816b87e8>] xs_tcp_state_change+0x78/0x320
[1122666.737643] [<ffffffff81667f4b>] tcp_fin+0x17b/0x210
[1122666.737717] [<ffffffff81668c52>] tcp_data_queue+0x4c2/0xe70
[1122666.737793] [<ffffffff8166c389>] ? tcp_validate_incoming+0x109/0x340
[1122666.737869] [<ffffffff8166c709>] tcp_rcv_established+0x149/0xb00
[1122666.737946] [<ffffffff816751dc>] tcp_v4_do_rcv+0x15c/0x2e0
[1122666.738021] [<ffffffff81649924>] ? nf_iterate+0x84/0xb0
[1122666.738096] [<ffffffff816759af>] tcp_v4_rcv+0x64f/0x8b0
[1122666.738170] [<ffffffff81649a9d>] ? nf_hook_slow+0x6d/0x130
[1122666.738246] [<ffffffff816543b0>] ? ip_rcv+0x2f0/0x2f0
[1122666.738322] [<ffffffff8165445a>] ip_local_deliver_finish+0xaa/0x1d0
[1122666.738399] [<ffffffff8165460d>] ip_local_deliver+0x8d/0xa0
[1122666.738474] [<ffffffff81653ef9>] ip_rcv_finish+0x169/0x330
[1122666.738549] [<ffffffff8165433d>] ip_rcv+0x27d/0x2f0
[1122666.738625] [<ffffffff816256e7>] __netif_receive_skb+0x437/0x490
[1122666.738701] [<ffffffff81627568>] netif_receive_skb+0x78/0x80
[1122666.738778] [<ffffffff813aa8dc>] ? is_swiotlb_buffer+0x3c/0x50
[1122666.738854] [<ffffffff816276a0>] napi_skb_finish+0x50/0x70
[1122666.738929] [<ffffffff81627ce5>] napi_gro_receive+0x105/0x140
[1122666.739008] [<ffffffffa0029e7b>] bnx2_poll_work+0x60b/0x1550 [bnx2]
[1122666.739085] [<ffffffff8105096d>] ? enqueue_task_fair+0x15d/0x540
[1122666.739163] [<ffffffff8104825e>] ? cpu_smt_mask+0x1e/0x20
[1122666.739238] [<ffffffff810475dd>] ? check_preempt_curr+0x7d/0xa0
[1122666.739315] [<ffffffffa002aeec>] bnx2_poll+0x5c/0x230 [bnx2]
[1122666.739391] [<ffffffff81627f0b>] net_rx_action+0x11b/0x2e0
[1122666.739467] [<ffffffff81019276>] ? read_tsc+0x16/0x40
[1122666.739542] [<ffffffff81064fa0>] __do_softirq+0xe0/0x1d0
[1122666.739619] [<ffffffff8170c8ec>] call_softirq+0x1c/0x30
[1122666.739693] [<ffffffff810142b5>] do_softirq+0x65/0xa0
[1122666.739767] [<ffffffff81064cad>] irq_exit+0xad/0xe0
[1122666.739841] [<ffffffff81013a94>] do_IRQ+0x64/0xe0
[1122666.739914] [<ffffffff81702dee>] common_interrupt+0x6e/0x6e
[1122666.739988] <EOI> [<ffffffff8170a712>] ? system_call_fastpath+0x16/0x1b
[1122666.740127] ---[ end trace 18327738ff1711ee ]---
[1122666.740203] RPC: server initated shutdown -- state 8 conn 1 dead 0 zapped 1 sk_shutdown 1
No other messages after this -- we see the socket in CLOSE_WAIT:
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name Timer
tcp 0 0 10.10.52.11:801 10.10.52.225:2049 CLOSE_WAIT - off (0.00/0/0)
Here's the dmesg after "rpcdebug -m rpc -c all": http://0x.ca/sim/ref/3.2.10/dmesg
Where should I go from here? Is it valid that some processes should get
stuck in uninterruptible sleep in this case? Is it valid that the server
should hang up at all?
Cheers,
Simon-
T24gVHVlLCAyMDEyLTA0LTI0IGF0IDE3OjU2IC0wNzAwLCBTaW1vbiBLaXJieSB3cm90ZToNCj4g
SGVsbG8hDQo+IA0KPiBJJ3ZlIHJlcG9ydGVkIHRoaXMgYSBmZXcgdGltZXMgbm93LCBidXQgd2Ug
c3RpbGwga2VlcCBoaXR0aW5nIHRoZSBpc3N1ZS4NCj4gTkZTIGNsaWVudHMgaW4gdGhlIGNsdXN0
ZXIgb2NjYXNpb25hbGx5IGdldCBzdHVjayB3aXRoIHRoZSBzb2NrZXQgdG8gb25lDQo+IG9mIHRo
ZSBORlMgc2VydmVycyBpbiBDTE9TRV9XQUlUIHN0YXRlLCBhbmQgZG9uJ3QgcmVjb3Zlci4NCj4g
DQo+IFRoZSBORlMgc2VydmVycyAoa25mc2QpIHVzZWQgdG8gYmUgcnVubmluZyAyLjYuMzAgc2lu
Y2UgbmV3ZXIga2VybmVscyBoaXQNCj4gWEZTIGJ1Z3MsIGJ1dCBhcmUgYWxsIG5vdyBmaW5hbGx5
IHJ1bm5pbmcgMy4yLjEwIG9yIDMuMi4xNi4NCj4gDQo+IEkndmUgYWRkZWQgc29tZSBkZWJ1Z2dp
bmcgcHJpbnRrKClzIGFuZCBXQVJOKClzIHRvIHRoZSBrZXJuZWwgdG8gdHJ5IHRvDQo+IGZpZ3Vy
ZSBvdXQgd2hhdCBpcyBnb2luZyBvbi4gV2l0aG91dCBrbm93aW5nIG11Y2ggYWJvdXQgaG93IGl0
J3MgYWN0dWFsbHkNCj4gZXhwZWN0ZWQgdG8gd29yayA6KSwgdGhlc2Ugd2VyZSBtb3N0bHkgdG8g
ZmlndXJlIG91dCB3aHkgdGhlIHNvY2tldCBpcw0KPiBiZWluZyBjbG9zZWQuIEkgYWRkZWQgdGhl
IGZvbGxvd2luZyBvbiB0b3Agb2Ygd2l0aCBUcm9uZCdzIHN1bnJwYw0KPiBkZWJ1Z2dpbmcgcGF0
Y2g6DQo8c25pcD4NCj4gWzExMTkxOTYuNjQwMDE3XSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDEx
MCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTYuNzIwMDI2XSBuZnM6IFJQQyBjYWxsIHRpbWVv
dXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTYuODkyMDIxXSBuZnM6IFJQQyBjYWxs
IHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTguNTE2MDIyXSBuZnM6IFJQ
QyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkyMjAuNTc2MDI1XSBu
ZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTk4NzEuMDQ4
MDQwXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMjAy
NjYuNDY0MDIxXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4g
WzExMjAzNDEuNDcyMDIxXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3Rp
bmcNCj4gWzExMjIxMzkuMDA4MDIyXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nv
bm5lY3RpbmcNCj4gWzExMjIxNjYuMTEyMDQzXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCks
IGRpc2Nvbm5lY3RpbmcNCj4gWzExMjIzMDIuMzA0MDM3XSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQg
KDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMjI2NjYuNzM1NzY1XSAtLS0tLS0tLS0tLS1bIGN1
dCBoZXJlIF0tLS0tLS0tLS0tLS0NCj4gWzExMjI2NjYuNzM1ODU1XSBXQVJOSU5HOiBhdCBuZXQv
c3VucnBjL3hwcnQuYzo2NDAgeHBydF9mb3JjZV9kaXNjb25uZWN0KzB4NGEvMHhhMCgpDQo+IFsx
MTIyNjY2LjczNTk0NV0gSGFyZHdhcmUgbmFtZTogUG93ZXJFZGdlIDE5NTANCj4gWzExMjI2NjYu
NzM2MDI1XSB4cHJ0X2ZvcmNlX2Rpc2Nvbm5lY3QoKTogc2V0dGluZyBYUFJUX0NMT1NFX1dBSVQN
Cj4gWzExMjI2NjYuNzM2MDk5XSBNb2R1bGVzIGxpbmtlZCBpbjogaXBtaV9kZXZpbnRmIGlwbWlf
c2kgaXBtaV9tc2doYW5kbGVyIHh0X3JlY2VudCBuZl9jb25udHJhY2tfZnRwIHh0X3N0YXRlIHh0
X293bmVyIG5mX2Nvbm50cmFja19pcHY0IG5mX2Nvbm50cmFjayBuZl9kZWZyYWdfaXB2NCBibngy
IHNlcmlvX3JhdyBldmRldg0KPiBbMTEyMjY2Ni43MzY4OThdIFBpZDogMTg0NTgsIGNvbW06IGh0
dHBkIFRhaW50ZWQ6IEcgICAgICAgIFcgICAgMy4yLjEwLWh3KyAjMTYNCj4gWzExMjI2NjYuNzM2
OTc1XSBDYWxsIFRyYWNlOg0KPiBbMTEyMjY2Ni43MzcwNDNdICA8SVJRPiAgWzxmZmZmZmZmZjgx
NmI1NDVhPl0gPyB4cHJ0X2ZvcmNlX2Rpc2Nvbm5lY3QrMHg0YS8weGEwDQo+IFsxMTIyNjY2Ljcz
NzE3Nl0gIFs8ZmZmZmZmZmY4MTA1ZGQ2MD5dIHdhcm5fc2xvd3BhdGhfY29tbW9uKzB4ODAvMHhj
MA0KPiBbMTEyMjY2Ni43MzcyNTJdICBbPGZmZmZmZmZmODEwNWRlOTk+XSB3YXJuX3Nsb3dwYXRo
X2ZtdCsweDY5LzB4NzANCj4gWzExMjI2NjYuNzM3MzM1XSAgWzxmZmZmZmZmZmEwMDU1YWZmPl0g
PyB0Y3BfcGFja2V0KzB4N2NmLzB4MTBmMCBbbmZfY29ubnRyYWNrXQ0KPiBbMTEyMjY2Ni43Mzc0
MTRdICBbPGZmZmZmZmZmODE3MDJjNDE+XSA/IF9yYXdfc3Bpbl9sb2NrX2JoKzB4MTEvMHg0MA0K
PiBbMTEyMjY2Ni43Mzc0OTBdICBbPGZmZmZmZmZmODE2YjU0NWE+XSB4cHJ0X2ZvcmNlX2Rpc2Nv
bm5lY3QrMHg0YS8weGEwDQo+IFsxMTIyNjY2LjczNzU2Nl0gIFs8ZmZmZmZmZmY4MTZiODdlOD5d
IHhzX3RjcF9zdGF0ZV9jaGFuZ2UrMHg3OC8weDMyMA0KPiBbMTEyMjY2Ni43Mzc2NDNdICBbPGZm
ZmZmZmZmODE2NjdmNGI+XSB0Y3BfZmluKzB4MTdiLzB4MjEwDQo+IFsxMTIyNjY2LjczNzcxN10g
IFs8ZmZmZmZmZmY4MTY2OGM1Mj5dIHRjcF9kYXRhX3F1ZXVlKzB4NGMyLzB4ZTcwDQo+IFsxMTIy
NjY2LjczNzc5M10gIFs8ZmZmZmZmZmY4MTY2YzM4OT5dID8gdGNwX3ZhbGlkYXRlX2luY29taW5n
KzB4MTA5LzB4MzQwDQo+IFsxMTIyNjY2LjczNzg2OV0gIFs8ZmZmZmZmZmY4MTY2YzcwOT5dIHRj
cF9yY3ZfZXN0YWJsaXNoZWQrMHgxNDkvMHhiMDANCj4gWzExMjI2NjYuNzM3OTQ2XSAgWzxmZmZm
ZmZmZjgxNjc1MWRjPl0gdGNwX3Y0X2RvX3JjdisweDE1Yy8weDJlMA0KPiBbMTEyMjY2Ni43Mzgw
MjFdICBbPGZmZmZmZmZmODE2NDk5MjQ+XSA/IG5mX2l0ZXJhdGUrMHg4NC8weGIwDQo+IFsxMTIy
NjY2LjczODA5Nl0gIFs8ZmZmZmZmZmY4MTY3NTlhZj5dIHRjcF92NF9yY3YrMHg2NGYvMHg4YjAN
Cj4gWzExMjI2NjYuNzM4MTcwXSAgWzxmZmZmZmZmZjgxNjQ5YTlkPl0gPyBuZl9ob29rX3Nsb3cr
MHg2ZC8weDEzMA0KPiBbMTEyMjY2Ni43MzgyNDZdICBbPGZmZmZmZmZmODE2NTQzYjA+XSA/IGlw
X3JjdisweDJmMC8weDJmMA0KPiBbMTEyMjY2Ni43MzgzMjJdICBbPGZmZmZmZmZmODE2NTQ0NWE+
XSBpcF9sb2NhbF9kZWxpdmVyX2ZpbmlzaCsweGFhLzB4MWQwDQo+IFsxMTIyNjY2LjczODM5OV0g
IFs8ZmZmZmZmZmY4MTY1NDYwZD5dIGlwX2xvY2FsX2RlbGl2ZXIrMHg4ZC8weGEwDQo+IFsxMTIy
NjY2LjczODQ3NF0gIFs8ZmZmZmZmZmY4MTY1M2VmOT5dIGlwX3Jjdl9maW5pc2grMHgxNjkvMHgz
MzANCj4gWzExMjI2NjYuNzM4NTQ5XSAgWzxmZmZmZmZmZjgxNjU0MzNkPl0gaXBfcmN2KzB4Mjdk
LzB4MmYwDQo+IFsxMTIyNjY2LjczODYyNV0gIFs8ZmZmZmZmZmY4MTYyNTZlNz5dIF9fbmV0aWZf
cmVjZWl2ZV9za2IrMHg0MzcvMHg0OTANCj4gWzExMjI2NjYuNzM4NzAxXSAgWzxmZmZmZmZmZjgx
NjI3NTY4Pl0gbmV0aWZfcmVjZWl2ZV9za2IrMHg3OC8weDgwDQo+IFsxMTIyNjY2LjczODc3OF0g
IFs8ZmZmZmZmZmY4MTNhYThkYz5dID8gaXNfc3dpb3RsYl9idWZmZXIrMHgzYy8weDUwDQo+IFsx
MTIyNjY2LjczODg1NF0gIFs8ZmZmZmZmZmY4MTYyNzZhMD5dIG5hcGlfc2tiX2ZpbmlzaCsweDUw
LzB4NzANCj4gWzExMjI2NjYuNzM4OTI5XSAgWzxmZmZmZmZmZjgxNjI3Y2U1Pl0gbmFwaV9ncm9f
cmVjZWl2ZSsweDEwNS8weDE0MA0KPiBbMTEyMjY2Ni43MzkwMDhdICBbPGZmZmZmZmZmYTAwMjll
N2I+XSBibngyX3BvbGxfd29yaysweDYwYi8weDE1NTAgW2JueDJdDQo+IFsxMTIyNjY2LjczOTA4
NV0gIFs8ZmZmZmZmZmY4MTA1MDk2ZD5dID8gZW5xdWV1ZV90YXNrX2ZhaXIrMHgxNWQvMHg1NDAN
Cj4gWzExMjI2NjYuNzM5MTYzXSAgWzxmZmZmZmZmZjgxMDQ4MjVlPl0gPyBjcHVfc210X21hc2sr
MHgxZS8weDIwDQo+IFsxMTIyNjY2LjczOTIzOF0gIFs8ZmZmZmZmZmY4MTA0NzVkZD5dID8gY2hl
Y2tfcHJlZW1wdF9jdXJyKzB4N2QvMHhhMA0KPiBbMTEyMjY2Ni43MzkzMTVdICBbPGZmZmZmZmZm
YTAwMmFlZWM+XSBibngyX3BvbGwrMHg1Yy8weDIzMCBbYm54Ml0NCj4gWzExMjI2NjYuNzM5Mzkx
XSAgWzxmZmZmZmZmZjgxNjI3ZjBiPl0gbmV0X3J4X2FjdGlvbisweDExYi8weDJlMA0KPiBbMTEy
MjY2Ni43Mzk0NjddICBbPGZmZmZmZmZmODEwMTkyNzY+XSA/IHJlYWRfdHNjKzB4MTYvMHg0MA0K
PiBbMTEyMjY2Ni43Mzk1NDJdICBbPGZmZmZmZmZmODEwNjRmYTA+XSBfX2RvX3NvZnRpcnErMHhl
MC8weDFkMA0KPiBbMTEyMjY2Ni43Mzk2MTldICBbPGZmZmZmZmZmODE3MGM4ZWM+XSBjYWxsX3Nv
ZnRpcnErMHgxYy8weDMwDQo+IFsxMTIyNjY2LjczOTY5M10gIFs8ZmZmZmZmZmY4MTAxNDJiNT5d
IGRvX3NvZnRpcnErMHg2NS8weGEwDQo+IFsxMTIyNjY2LjczOTc2N10gIFs8ZmZmZmZmZmY4MTA2
NGNhZD5dIGlycV9leGl0KzB4YWQvMHhlMA0KPiBbMTEyMjY2Ni43Mzk4NDFdICBbPGZmZmZmZmZm
ODEwMTNhOTQ+XSBkb19JUlErMHg2NC8weGUwDQo+IFsxMTIyNjY2LjczOTkxNF0gIFs8ZmZmZmZm
ZmY4MTcwMmRlZT5dIGNvbW1vbl9pbnRlcnJ1cHQrMHg2ZS8weDZlDQo+IFsxMTIyNjY2LjczOTk4
OF0gIDxFT0k+ICBbPGZmZmZmZmZmODE3MGE3MTI+XSA/IHN5c3RlbV9jYWxsX2Zhc3RwYXRoKzB4
MTYvMHgxYg0KPiBbMTEyMjY2Ni43NDAxMjddIC0tLVsgZW5kIHRyYWNlIDE4MzI3NzM4ZmYxNzEx
ZWUgXS0tLQ0KPiBbMTEyMjY2Ni43NDAyMDNdIFJQQzogc2VydmVyIGluaXRhdGVkIHNodXRkb3du
IC0tIHN0YXRlIDggY29ubiAxIGRlYWQgMCB6YXBwZWQgMSBza19zaHV0ZG93biAxDQo+IA0KPiBO
byBvdGhlciBtZXNzYWdlcyBhZnRlciB0aGlzIC0tIHdlIHNlZSB0aGUgc29ja2V0IGluIENMT1NF
X1dBSVQ6DQo+IA0KPiBQcm90byBSZWN2LVEgU2VuZC1RIExvY2FsIEFkZHJlc3MgICAgICAgICAg
IEZvcmVpZ24gQWRkcmVzcyAgICAgICAgIFN0YXRlICAgICAgIFBJRC9Qcm9ncmFtIG5hbWUgVGlt
ZXINCj4gdGNwICAgICAgICAwICAgICAgMCAxMC4xMC41Mi4xMTo4MDEgICAgICAgICAxMC4xMC41
Mi4yMjU6MjA0OSAgICAgICBDTE9TRV9XQUlUICAtICAgICAgICAgICAgICAgIG9mZiAoMC4wMC8w
LzApDQo+IA0KPiBIZXJlJ3MgdGhlIGRtZXNnIGFmdGVyICJycGNkZWJ1ZyAtbSBycGMgLWMgYWxs
IjogaHR0cDovLzB4LmNhL3NpbS9yZWYvMy4yLjEwL2RtZXNnDQo+IA0KPiBXaGVyZSBzaG91bGQg
SSBnbyBmcm9tIGhlcmU/IElzIGl0IHZhbGlkIHRoYXQgc29tZSBwcm9jZXNzZXMgc2hvdWxkIGdl
dA0KPiBzdHVjayBpbiB1bmludGVycnVwdGlibGUgc2xlZXAgaW4gdGhpcyBjYXNlPyBJcyBpdCB2
YWxpZCB0aGF0IHRoZSBzZXJ2ZXINCj4gc2hvdWxkIGhhbmcgdXAgYXQgYWxsPw0KDQpDYW4geW91
IHRyeSBhcHBseWluZyBzb21ldGhpbmcgbGlrZSB0aGUgZm9sbG93aW5nIHBhdGNoIG9uIHRvcCBv
ZiB3aGF0DQp5b3UgaGF2ZSBhcHBsaWVkIGFib3ZlPyBJdCBpcyBjbGVhciB0aGF0IHNvbWV0aGlu
ZyBpcyBnb2luZyB3cm9uZyBpbiB0aGUNCmF1dG9kaXNjb25uZWN0IGxvZ2ljLCBidXQgaXQgaXMg
bm90IHlldCAxMDAlIGNsZWFyIHRvIG1lIHdoYXQgdGhhdCBtaWdodA0KYmUuIFRoZSBwYXRjaCBi
ZWxvdyBjaGVja3MgZm9yIHJhY2VzIGluIHhwcnRfYXV0b2Nsb3NlKCkgYW5kIGFsc28gY2hlY2tz
DQp0byBzZWUgaWYgdGhlIHhwcnRfZm9yY2VfZGlzY29ubmVjdCBtaWdodCBiZSBmYWlsaW5nIHRv
IHdha2UgdXAgYSBzdHVjaw0KdGFzayB0aGF0IGhvbGRzIHRoZSBzb2NrZXQgd3JpdGUgbG9jay4N
Cg0KQ2hlZXJzDQogIFRyb25kDQotLS0NCmRpZmYgLS1naXQgYS9uZXQvc3VucnBjL3hwcnQuYyBi
L25ldC9zdW5ycGMveHBydC5jDQppbmRleCBjNjRjMGVmLi4xOTA0NDY3IDEwMDY0NA0KLS0tIGEv
bmV0L3N1bnJwYy94cHJ0LmMNCisrKyBiL25ldC9zdW5ycGMveHBydC5jDQpAQCAtNjA0LDggKzYw
NCw5IEBAIHN0YXRpYyB2b2lkIHhwcnRfYXV0b2Nsb3NlKHN0cnVjdCB3b3JrX3N0cnVjdCAqd29y
aykNCiAJc3RydWN0IHJwY194cHJ0ICp4cHJ0ID0NCiAJCWNvbnRhaW5lcl9vZih3b3JrLCBzdHJ1
Y3QgcnBjX3hwcnQsIHRhc2tfY2xlYW51cCk7DQogDQotCXhwcnQtPm9wcy0+Y2xvc2UoeHBydCk7
DQogCWNsZWFyX2JpdChYUFJUX0NMT1NFX1dBSVQsICZ4cHJ0LT5zdGF0ZSk7DQorCXhwcnQtPm9w
cy0+Y2xvc2UoeHBydCk7DQorCVdBUk5fT04odGVzdF9iaXQoWFBSVF9DTE9TRV9XQUlULCAmeHBy
dC0+c3RhdGUpKTsNCiAJeHBydF9yZWxlYXNlX3dyaXRlKHhwcnQsIE5VTEwpOw0KIH0NCiANCkBA
IC02MzgsNiArNjM5LDE0IEBAIHZvaWQgeHBydF9mb3JjZV9kaXNjb25uZWN0KHN0cnVjdCBycGNf
eHBydCAqeHBydCkNCiAJaWYgKHRlc3RfYW5kX3NldF9iaXQoWFBSVF9MT0NLRUQsICZ4cHJ0LT5z
dGF0ZSkgPT0gMCkNCiAJCXF1ZXVlX3dvcmsocnBjaW9kX3dvcmtxdWV1ZSwgJnhwcnQtPnRhc2tf
Y2xlYW51cCk7DQogCXhwcnRfd2FrZV9wZW5kaW5nX3Rhc2tzKHhwcnQsIC1FQUdBSU4pOw0KKwlp
ZiAoeHBydC0+c25kX3Rhc2sgIT0gTlVMTCkgew0KKwkJc3RydWN0IHJwY190YXNrICp0YXNrID0g
eHBydC0+c25kX3Rhc2s7DQorCQlpZiAoUlBDX0lTX1FVRVVFRCh0YXNrKSkgew0KKwkJCVdBUk4o
MSwgInRhc2sgJTV1IGlzIHF1ZXVlZCBvbiAlc1xuIiwNCisJCQkJCXRhc2stPnRrX3BpZCwNCisJ
CQkJCXJwY19xbmFtZSh0YXNrLT50a193YWl0cXVldWUpKTsNCisJCX0NCisJfQ0KIAlzcGluX3Vu
bG9ja19iaCgmeHBydC0+dHJhbnNwb3J0X2xvY2spOw0KIH0NCiANCg0KLS0gDQpUcm9uZCBNeWts
ZWJ1c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0KDQpOZXRBcHANClRyb25kLk15a2xl
YnVzdEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=