From: Andy Adamson <[email protected]>
Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible"
uses the nfs_client cl_rpcclient for all state management operations, and
will use krb5i or auth_sys with no regard to the mount command authflavor
choice.
The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all
non-state management operations with a different nfs_server for each fsid
encountered traversing the mount point, each with a potentially different
auth flavor.
pNFS data servers are not mounted in the normal sense as there is no associated
nfs_server structure. Data servers can also export multiple fsids, each with
a potentially different auth flavor.
Data servers need to use the same authflavor as the MDS server rpc client for
non-state management operations. Populate a list of rpc clients with the MDS
server rpc client auth flavor for the DS to use.
Signed-off-by: Andy Adamson <[email protected]>
---
fs/nfs/internal.h | 2 +
fs/nfs/nfs4client.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4filelayout.c | 35 +++++++++++----
include/linux/nfs_fs_sb.h | 1 +
4 files changed, 140 insertions(+), 8 deletions(-)
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 2415198..23ec6e8 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -186,6 +186,8 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
int ds_addrlen, int ds_proto,
unsigned int ds_timeo,
unsigned int ds_retrans);
+extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
+ struct inode *);
#ifdef CONFIG_PROC_FS
extern int __init nfs_fs_proc_init(void);
extern void nfs_fs_proc_exit(void);
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index 767a5e3..a8dd396 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -49,10 +49,118 @@ static void nfs4_shutdown_session(struct nfs_client *clp)
}
}
+
+/**
+ * Per auth flavor data server rpc clients
+ */
+struct nfs4_ds_server {
+ struct list_head list; /* ds_clp->cl_ds_clients */
+ struct rpc_clnt *rpc_clnt;
+};
+
+static struct nfs4_ds_server *
+nfs4_find_or_add_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor,
+ struct nfs4_ds_server *new)
+{
+ struct nfs4_ds_server *dss, *tmp;
+
+ spin_lock(&ds_clp->cl_lock);
+ list_for_each_entry_safe(dss, tmp, &ds_clp->cl_ds_clients, list) {
+ if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
+ continue;
+ goto out;
+ }
+ if (new)
+ list_add(&new->list, &ds_clp->cl_ds_clients);
+ dss = new;
+out:
+ spin_unlock(&ds_clp->cl_lock); /* need some lock to protect list */
+ return dss;
+}
+
+static struct nfs4_ds_server *
+nfs4_alloc_ds_server(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
+{
+ struct nfs4_ds_server *dss;
+
+ dss = kmalloc(sizeof(*dss), GFP_NOFS);
+ if (dss == NULL)
+ return ERR_PTR(-ENOMEM);
+
+ dss->rpc_clnt = rpc_clone_client_set_auth(ds_clp->cl_rpcclient, flavor);
+ if (IS_ERR(dss->rpc_clnt)) {
+ int err = PTR_ERR(dss->rpc_clnt);
+ kfree (dss);
+ return ERR_PTR(err);
+ }
+ INIT_LIST_HEAD(&dss->list);
+
+ return dss;
+}
+
+static void
+nfs4_free_ds_server(struct nfs4_ds_server *dss)
+{
+ rpc_release_client(dss->rpc_clnt);
+ kfree(dss);
+}
+
+/**
+ * Find or create a DS rpc client with th MDS server rpc client auth flavor
+ * in the nfs_client cl_ds_clients list.
+ */
+struct rpc_clnt *
+nfs4_find_or_create_ds_client(struct nfs_client *ds_clp, struct inode *inode)
+{
+ struct nfs4_ds_server *dss, *new;
+ rpc_authflavor_t flavor = NFS_SERVER(inode)->client->cl_auth->au_flavor;
+
+ dss = nfs4_find_or_add_ds_client(ds_clp, flavor, NULL);
+ if (dss != NULL)
+ goto out;
+ new = nfs4_alloc_ds_server(ds_clp, flavor);
+ if (IS_ERR(new))
+ return (struct rpc_clnt *)new;
+ dss = nfs4_find_or_add_ds_client(ds_clp, flavor, new);
+ if (dss != new)
+ nfs4_free_ds_server(new);
+out:
+ return dss->rpc_clnt;
+}
+EXPORT_SYMBOL_GPL(nfs4_find_or_create_ds_client);
+
+static void
+nfs4_shutdown_ds_clients(struct nfs_client *clp)
+{
+ struct nfs4_ds_server *dss;
+ LIST_HEAD(shutdown_list);
+
+ spin_lock(&clp->cl_lock);
+ while (!list_empty(&clp->cl_ds_clients)) {
+ dss = list_entry(clp->cl_ds_clients.next,
+ struct nfs4_ds_server, list);
+ list_move(&dss->list, &shutdown_list);
+ }
+ spin_unlock(&clp->cl_lock);
+
+ while (!list_empty(&shutdown_list)) {
+ dss = list_entry(shutdown_list.next,
+ struct nfs4_ds_server, list);
+ list_del(&dss->list);
+ rpc_shutdown_client(dss->rpc_clnt);
+ kfree (dss);
+ }
+}
+
#else /* CONFIG_NFS_V4_1 */
static void nfs4_shutdown_session(struct nfs_client *clp)
{
}
+
+static void
+nfs4_shutdown_ds_clients(struct nfs_client *clp)
+{
+}
#endif /* CONFIG_NFS_V4_1 */
struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
@@ -73,6 +181,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
spin_lock_init(&clp->cl_lock);
INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
+ INIT_LIST_HEAD(&clp->cl_ds_clients);
rpc_init_wait_queue(&clp->cl_rpcwaitq, "NFS client");
clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED;
clp->cl_minorversion = cl_init->minorversion;
@@ -97,6 +206,7 @@ static void nfs4_shutdown_client(struct nfs_client *clp)
{
if (__test_and_clear_bit(NFS_CS_RENEWD, &clp->cl_res_state))
nfs4_kill_renewd(clp);
+ nfs4_shutdown_ds_clients(clp);
nfs4_shutdown_session(clp);
nfs4_destroy_callback(clp);
if (__test_and_clear_bit(NFS_CS_IDMAP, &clp->cl_res_state))
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index a70cb3a..b86464b 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -528,6 +528,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
struct nfs_pgio_header *hdr = data->header;
struct pnfs_layout_segment *lseg = hdr->lseg;
struct nfs4_pnfs_ds *ds;
+ struct rpc_clnt *ds_clnt;
loff_t offset = data->args.offset;
u32 j, idx;
struct nfs_fh *fh;
@@ -542,6 +543,11 @@ filelayout_read_pagelist(struct nfs_read_data *data)
ds = nfs4_fl_prepare_ds(lseg, idx);
if (!ds)
return PNFS_NOT_ATTEMPTED;
+
+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+ if (IS_ERR(ds_clnt))
+ return PNFS_NOT_ATTEMPTED;
+
dprintk("%s USE DS: %s cl_count %d\n", __func__,
ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
@@ -556,7 +562,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
data->mds_offset = offset;
/* Perform an asynchronous read to ds */
- nfs_initiate_read(ds->ds_clp->cl_rpcclient, data,
+ nfs_initiate_read(ds_clnt, data,
&filelayout_read_call_ops, RPC_TASK_SOFTCONN);
return PNFS_ATTEMPTED;
}
@@ -568,6 +574,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
struct nfs_pgio_header *hdr = data->header;
struct pnfs_layout_segment *lseg = hdr->lseg;
struct nfs4_pnfs_ds *ds;
+ struct rpc_clnt *ds_clnt;
loff_t offset = data->args.offset;
u32 j, idx;
struct nfs_fh *fh;
@@ -578,6 +585,11 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
ds = nfs4_fl_prepare_ds(lseg, idx);
if (!ds)
return PNFS_NOT_ATTEMPTED;
+
+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+ if (IS_ERR(ds_clnt))
+ return PNFS_NOT_ATTEMPTED;
+
dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d\n",
__func__, hdr->inode->i_ino, sync, (size_t) data->args.count,
offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
@@ -595,7 +607,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
data->args.offset = filelayout_get_dserver_offset(lseg, offset);
/* Perform an asynchronous write */
- nfs_initiate_write(ds->ds_clp->cl_rpcclient, data,
+ nfs_initiate_write(ds_clnt, data,
&filelayout_write_call_ops, sync,
RPC_TASK_SOFTCONN);
return PNFS_ATTEMPTED;
@@ -1105,16 +1117,19 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
{
struct pnfs_layout_segment *lseg = data->lseg;
struct nfs4_pnfs_ds *ds;
+ struct rpc_clnt *ds_clnt;
u32 idx;
struct nfs_fh *fh;
idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
ds = nfs4_fl_prepare_ds(lseg, idx);
- if (!ds) {
- prepare_to_resend_writes(data);
- filelayout_commit_release(data);
- return -EAGAIN;
- }
+ if (!ds)
+ goto out_err;
+
+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode);
+ if (IS_ERR(ds_clnt))
+ goto out_err;
+
dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count));
data->commit_done_cb = filelayout_commit_done_cb;
@@ -1123,9 +1138,13 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
if (fh)
data->args.fh = fh;
- return nfs_initiate_commit(ds->ds_clp->cl_rpcclient, data,
+ return nfs_initiate_commit(ds_clnt, data,
&filelayout_commit_call_ops, how,
RPC_TASK_SOFTCONN);
+out_err:
+ prepare_to_resend_writes(data);
+ filelayout_commit_release(data);
+ return -EAGAIN;
}
static int
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index d221243..4d476e7 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -56,6 +56,7 @@ struct nfs_client {
struct rpc_cred *cl_machine_cred;
#if IS_ENABLED(CONFIG_NFS_V4)
+ struct list_head cl_ds_clients; /* auth flavor data servers */
u64 cl_clientid; /* constant */
nfs4_verifier cl_confirm; /* Clientid verifier */
unsigned long cl_state;
--
1.8.3.1
T24gVGh1LCAyMDEzLTA5LTA1IGF0IDE4OjIxICswMDAwLCBBZGFtc29uLCBBbmR5IHdyb3RlOg0K
PiBPbiBTZXAgNSwgMjAxMywgYXQgMTo0OCBQTSwgIk15a2xlYnVzdCwgVHJvbmQiIDxUcm9uZC5N
eWtsZWJ1c3RAbmV0YXBwLmNvbT4NCj4gIHdyb3RlOg0KPiA+PiArc3RhdGljIHZvaWQNCj4gPj4g
K25mczRfc2h1dGRvd25fZHNfY2xpZW50cyhzdHJ1Y3QgbmZzX2NsaWVudCAqY2xwKQ0KPiA+PiAr
ew0KPiA+PiArCXN0cnVjdCBuZnM0X2RzX3NlcnZlciAqZHNzOw0KPiA+PiArCUxJU1RfSEVBRChz
aHV0ZG93bl9saXN0KTsNCj4gPj4gKw0KPiA+PiArCXNwaW5fbG9jaygmY2xwLT5jbF9sb2NrKTsN
Cj4gPj4gKwl3aGlsZSAoIWxpc3RfZW1wdHkoJmNscC0+Y2xfZHNfY2xpZW50cykpIHsNCj4gPj4g
KwkJZHNzID0gbGlzdF9lbnRyeShjbHAtPmNsX2RzX2NsaWVudHMubmV4dCwNCj4gPj4gKwkJCQkg
IHN0cnVjdCBuZnM0X2RzX3NlcnZlciwgbGlzdCk7DQo+ID4+ICsJCWxpc3RfbW92ZSgmZHNzLT5s
aXN0LCAmc2h1dGRvd25fbGlzdCk7DQo+ID4gDQo+ID4gSXMgdGhpcyBzdGVwIG5lY2Vzc2FyeT8g
V2Uga25vdyB0aGF0IG5vYm9keSBvdGhlciB0aGFuIHVzIGlzIHJlZmVyZW5jaW5nDQo+ID4gdGhl
IG5mc19jbGllbnQgYXQgdGhpcyBwb2ludC4NCj4gDQo+IEJlY2F1c2Ugb2YgdGhlIGtmcmVlIHVu
ZGVyIHRoZSBzcGluIGxvY2sgLSBidXQgcGVyaGFwcyB0aGF0IGlzIG5vIGxvbmdlciBhbiBpc3N1
ZT8NCg0KRG8geW91IG5lZWQgdGhlIHNwaW4gbG9jayBhdCBhbGwgaGVyZT8gTm8gb3RoZXIgdGhy
ZWFkcyBhcmUgc3VwcG9zZWQgdG8NCmJlIGFjY2Vzc2luZyB0aGUgbmZzX2NsaWVudCB3aGVuIHdl
IGNhbGwgLT5mcmVlX2NsaWVudCgpLg0KDQotLSANClRyb25kIE15a2xlYnVzdA0KTGludXggTkZT
IGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFwcA0KVHJvbmQuTXlrbGVidXN0QG5ldGFwcC5jb20N
Cnd3dy5uZXRhcHAuY29tDQo=
T24gVGh1LCAyMDEzLTA5LTA1IGF0IDEzOjMwIC0wNDAwLCBhbmRyb3NAbmV0YXBwLmNvbSB3cm90
ZToNCj4gRnJvbTogQW5keSBBZGFtc29uIDxhbmRyb3NAbmV0YXBwLmNvbT4NCj4gDQo+IENvbW1p
dCA0ZWRhYTMwOCAiTkZTOiBVc2UgImtyYjVpIiB0byBlc3RhYmxpc2ggTkZTdjQgc3RhdGUgd2hl
bmV2ZXIgcG9zc2libGUiDQo+IHVzZXMgdGhlIG5mc19jbGllbnQgY2xfcnBjY2xpZW50IGZvciBh
bGwgc3RhdGUgbWFuYWdlbWVudCBvcGVyYXRpb25zLCBhbmQNCj4gd2lsbCB1c2Uga3JiNWkgb3Ig
YXV0aF9zeXMgd2l0aCBubyByZWdhcmQgdG8gdGhlIG1vdW50IGNvbW1hbmQgYXV0aGZsYXZvcg0K
PiBjaG9pY2UuDQo+IA0KPiBUaGUgTURTLCBhcyBhbnkgTkZTdjQuMSBtb3VudCBwb2ludCwgdXNl
cyB0aGUgbmZzX3NlcnZlciBycGMgY2xpZW50IGZvciBhbGwNCj4gbm9uLXN0YXRlIG1hbmFnZW1l
bnQgb3BlcmF0aW9ucyB3aXRoIGEgZGlmZmVyZW50IG5mc19zZXJ2ZXIgZm9yIGVhY2ggZnNpZA0K
PiBlbmNvdW50ZXJlZCB0cmF2ZXJzaW5nIHRoZSBtb3VudCBwb2ludCwgZWFjaCB3aXRoIGEgcG90
ZW50aWFsbHkgZGlmZmVyZW50DQo+IGF1dGggZmxhdm9yLg0KPiANCj4gcE5GUyBkYXRhIHNlcnZl
cnMgYXJlIG5vdCBtb3VudGVkIGluIHRoZSBub3JtYWwgc2Vuc2UgYXMgdGhlcmUgaXMgbm8gYXNz
b2NpYXRlZA0KPiBuZnNfc2VydmVyIHN0cnVjdHVyZS4gRGF0YSBzZXJ2ZXJzIGNhbiBhbHNvIGV4
cG9ydCBtdWx0aXBsZSBmc2lkcywgZWFjaCB3aXRoDQo+IGEgcG90ZW50aWFsbHkgZGlmZmVyZW50
IGF1dGggZmxhdm9yLg0KPiANCj4gRGF0YSBzZXJ2ZXJzIG5lZWQgdG8gdXNlIHRoZSBzYW1lIGF1
dGhmbGF2b3IgYXMgdGhlIE1EUyBzZXJ2ZXIgcnBjIGNsaWVudCBmb3INCj4gbm9uLXN0YXRlIG1h
bmFnZW1lbnQgb3BlcmF0aW9ucy4gUG9wdWxhdGUgYSBsaXN0IG9mIHJwYyBjbGllbnRzIHdpdGgg
dGhlIE1EUw0KPiBzZXJ2ZXIgcnBjIGNsaWVudCBhdXRoIGZsYXZvciBmb3IgdGhlIERTIHRvIHVz
ZS4NCj4gDQo+IFNpZ25lZC1vZmYtYnk6IEFuZHkgQWRhbXNvbiA8YW5kcm9zQG5ldGFwcC5jb20+
DQo+IC0tLQ0KPiAgZnMvbmZzL2ludGVybmFsLmggICAgICAgICB8ICAgMiArDQo+ICBmcy9uZnMv
bmZzNGNsaWVudC5jICAgICAgIHwgMTEwICsrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysr
KysrKysrKysrKysrKysNCj4gIGZzL25mcy9uZnM0ZmlsZWxheW91dC5jICAgfCAgMzUgKysrKysr
KysrKystLS0tDQo+ICBpbmNsdWRlL2xpbnV4L25mc19mc19zYi5oIHwgICAxICsNCj4gIDQgZmls
ZXMgY2hhbmdlZCwgMTQwIGluc2VydGlvbnMoKyksIDggZGVsZXRpb25zKC0pDQoNClRoYXQgbG9v
a3MgYSBsb3QgYmV0dGVyLiBTZWUgY29tbWVudHMgaW5saW5lIGJlbG93Lg0KDQo+IGRpZmYgLS1n
aXQgYS9mcy9uZnMvaW50ZXJuYWwuaCBiL2ZzL25mcy9pbnRlcm5hbC5oDQo+IGluZGV4IDI0MTUx
OTguLjIzZWM2ZTggMTAwNjQ0DQo+IC0tLSBhL2ZzL25mcy9pbnRlcm5hbC5oDQo+ICsrKyBiL2Zz
L25mcy9pbnRlcm5hbC5oDQo+IEBAIC0xODYsNiArMTg2LDggQEAgZXh0ZXJuIHN0cnVjdCBuZnNf
Y2xpZW50ICpuZnM0X3NldF9kc19jbGllbnQoc3RydWN0IG5mc19jbGllbnQqIG1kc19jbHAsDQo+
ICAJCQkJCSAgICAgaW50IGRzX2FkZHJsZW4sIGludCBkc19wcm90bywNCj4gIAkJCQkJICAgICB1
bnNpZ25lZCBpbnQgZHNfdGltZW8sDQo+ICAJCQkJCSAgICAgdW5zaWduZWQgaW50IGRzX3JldHJh
bnMpOw0KPiArZXh0ZXJuIHN0cnVjdCBycGNfY2xudCAqbmZzNF9maW5kX29yX2NyZWF0ZV9kc19j
bGllbnQoc3RydWN0IG5mc19jbGllbnQgKiwNCj4gKwkJCQkJCXN0cnVjdCBpbm9kZSAqKTsNCj4g
ICNpZmRlZiBDT05GSUdfUFJPQ19GUw0KPiAgZXh0ZXJuIGludCBfX2luaXQgbmZzX2ZzX3Byb2Nf
aW5pdCh2b2lkKTsNCj4gIGV4dGVybiB2b2lkIG5mc19mc19wcm9jX2V4aXQodm9pZCk7DQo+IGRp
ZmYgLS1naXQgYS9mcy9uZnMvbmZzNGNsaWVudC5jIGIvZnMvbmZzL25mczRjbGllbnQuYw0KPiBp
bmRleCA3NjdhNWUzLi5hOGRkMzk2IDEwMDY0NA0KPiAtLS0gYS9mcy9uZnMvbmZzNGNsaWVudC5j
DQo+ICsrKyBiL2ZzL25mcy9uZnM0Y2xpZW50LmMNCj4gQEAgLTQ5LDEwICs0OSwxMTggQEAgc3Rh
dGljIHZvaWQgbmZzNF9zaHV0ZG93bl9zZXNzaW9uKHN0cnVjdCBuZnNfY2xpZW50ICpjbHApDQo+
ICAJfQ0KPiAgDQo+ICB9DQo+ICsNCj4gKy8qKg0KPiArICogUGVyIGF1dGggZmxhdm9yIGRhdGEg
c2VydmVyIHJwYyBjbGllbnRzDQo+ICsgKi8NCj4gK3N0cnVjdCBuZnM0X2RzX3NlcnZlciB7DQo+
ICsJc3RydWN0IGxpc3RfaGVhZAlsaXN0OwkvKiBkc19jbHAtPmNsX2RzX2NsaWVudHMgKi8NCj4g
KwlzdHJ1Y3QgcnBjX2NsbnQJCSpycGNfY2xudDsNCj4gK307DQo+ICsNCj4gK3N0YXRpYyBzdHJ1
Y3QgbmZzNF9kc19zZXJ2ZXIgKg0KPiArbmZzNF9maW5kX29yX2FkZF9kc19jbGllbnQoc3RydWN0
IG5mc19jbGllbnQgKmRzX2NscCwgcnBjX2F1dGhmbGF2b3JfdCBmbGF2b3IsDQo+ICsJCQkgICBz
dHJ1Y3QgbmZzNF9kc19zZXJ2ZXIgKm5ldykNCj4gK3sNCj4gKwlzdHJ1Y3QgbmZzNF9kc19zZXJ2
ZXIgKmRzcywgKnRtcDsNCj4gKw0KPiArCXNwaW5fbG9jaygmZHNfY2xwLT5jbF9sb2NrKTsNCj4g
KwlsaXN0X2Zvcl9lYWNoX2VudHJ5X3NhZmUoZHNzLCB0bXAsICZkc19jbHAtPmNsX2RzX2NsaWVu
dHMsIGxpc3QpIHsNCg0KTml0OiB3aHkgZG8gd2UgbmVlZCBsaXN0X2Zvcl9lYWNoX2VudHJ5X3Nh
ZmUoKSBoZXJlPyBZb3UncmUgbm90IHJlbW92aW5nDQphbnl0aGluZyBmcm9tIHRoYXQgbGlzdC4N
Cg0KPiArCQlpZiAoZHNzLT5ycGNfY2xudC0+Y2xfYXV0aC0+YXVfZmxhdm9yICE9IGZsYXZvcikN
Cj4gKwkJCWNvbnRpbnVlOw0KPiArCQlnb3RvIG91dDsNCj4gKwl9DQo+ICsJaWYgKG5ldykNCj4g
KwkJbGlzdF9hZGQoJm5ldy0+bGlzdCwgJmRzX2NscC0+Y2xfZHNfY2xpZW50cyk7DQo+ICsJZHNz
ID0gbmV3Ow0KPiArb3V0Og0KPiArCXNwaW5fdW5sb2NrKCZkc19jbHAtPmNsX2xvY2spOyAvKiBu
ZWVkIHNvbWUgbG9jayB0byBwcm90ZWN0IGxpc3QgKi8NCj4gKwlyZXR1cm4gZHNzOw0KPiArfQ0K
PiArDQo+ICtzdGF0aWMgc3RydWN0IG5mczRfZHNfc2VydmVyICoNCj4gK25mczRfYWxsb2NfZHNf
c2VydmVyKHN0cnVjdCBuZnNfY2xpZW50ICpkc19jbHAsIHJwY19hdXRoZmxhdm9yX3QgZmxhdm9y
KQ0KPiArew0KPiArCXN0cnVjdCBuZnM0X2RzX3NlcnZlciAqZHNzOw0KPiArDQo+ICsJZHNzID0g
a21hbGxvYyhzaXplb2YoKmRzcyksIEdGUF9OT0ZTKTsNCj4gKwlpZiAoZHNzID09IE5VTEwpDQo+
ICsJCXJldHVybiBFUlJfUFRSKC1FTk9NRU0pOw0KPiArDQo+ICsJZHNzLT5ycGNfY2xudCA9IHJw
Y19jbG9uZV9jbGllbnRfc2V0X2F1dGgoZHNfY2xwLT5jbF9ycGNjbGllbnQsIGZsYXZvcik7DQo+
ICsJaWYgKElTX0VSUihkc3MtPnJwY19jbG50KSkgew0KPiArCQlpbnQgZXJyID0gUFRSX0VSUihk
c3MtPnJwY19jbG50KTsNCj4gKwkJa2ZyZWUgKGRzcyk7DQo+ICsJCXJldHVybiBFUlJfUFRSKGVy
cik7DQo+ICsJfQ0KPiArCUlOSVRfTElTVF9IRUFEKCZkc3MtPmxpc3QpOw0KPiArDQo+ICsJcmV0
dXJuIGRzczsNCj4gK30NCj4gKw0KPiArc3RhdGljIHZvaWQNCj4gK25mczRfZnJlZV9kc19zZXJ2
ZXIoc3RydWN0IG5mczRfZHNfc2VydmVyICpkc3MpDQo+ICt7DQo+ICsJcnBjX3JlbGVhc2VfY2xp
ZW50KGRzcy0+cnBjX2NsbnQpOw0KPiArCWtmcmVlKGRzcyk7DQo+ICt9DQo+ICsNCj4gKy8qKg0K
PiArICogRmluZCBvciBjcmVhdGUgYSBEUyBycGMgY2xpZW50IHdpdGggdGggTURTIHNlcnZlciBy
cGMgY2xpZW50IGF1dGggZmxhdm9yDQo+ICsgKiBpbiB0aGUgbmZzX2NsaWVudCBjbF9kc19jbGll
bnRzIGxpc3QuDQo+ICsgKi8NCj4gK3N0cnVjdCBycGNfY2xudCAqDQo+ICtuZnM0X2ZpbmRfb3Jf
Y3JlYXRlX2RzX2NsaWVudChzdHJ1Y3QgbmZzX2NsaWVudCAqZHNfY2xwLCBzdHJ1Y3QgaW5vZGUg
Kmlub2RlKQ0KPiArew0KPiArCXN0cnVjdCBuZnM0X2RzX3NlcnZlciAqZHNzLCAqbmV3Ow0KPiAr
CXJwY19hdXRoZmxhdm9yX3QgZmxhdm9yID0gTkZTX1NFUlZFUihpbm9kZSktPmNsaWVudC0+Y2xf
YXV0aC0+YXVfZmxhdm9yOw0KPiArDQo+ICsJZHNzID0gbmZzNF9maW5kX29yX2FkZF9kc19jbGll
bnQoZHNfY2xwLCBmbGF2b3IsIE5VTEwpOw0KDQpEb2VzIGl0IG1ha2Ugc2Vuc2UgdG8gcGVyaGFw
cyB1c2UgUkNVIGluIHRoaXMgY2FzZSAoYnV0IG5vdCB0aGUgb25lDQpiZWxvdykgaW4gb3JkZXIg
dG8gYXZvaWQgdGhlIGNvbnRlbnRpb24gb24gY2xwLT5jbF9sb2NrPw0KDQo+ICsJaWYgKGRzcyAh
PSBOVUxMKQ0KPiArCQlnb3RvIG91dDsNCj4gKwluZXcgPSBuZnM0X2FsbG9jX2RzX3NlcnZlcihk
c19jbHAsIGZsYXZvcik7DQo+ICsJaWYgKElTX0VSUihuZXcpKQ0KPiArCQlyZXR1cm4gKHN0cnVj
dCBycGNfY2xudCAqKW5ldzsNCg0KTml0OiB5b3Ugc2hvdWxkIHVzZSBFUlJfQ0FTVCgpIHJhdGhl
ciB0aGFuIGFuIGV4cGxpY2l0IGNhc3QgYWJvdmUNCg0KPiArCWRzcyA9IG5mczRfZmluZF9vcl9h
ZGRfZHNfY2xpZW50KGRzX2NscCwgZmxhdm9yLCBuZXcpOw0KPiArCWlmIChkc3MgIT0gbmV3KQ0K
PiArCQluZnM0X2ZyZWVfZHNfc2VydmVyKG5ldyk7DQo+ICtvdXQ6DQo+ICsJcmV0dXJuIGRzcy0+
cnBjX2NsbnQ7DQo+ICt9DQo+ICtFWFBPUlRfU1lNQk9MX0dQTChuZnM0X2ZpbmRfb3JfY3JlYXRl
X2RzX2NsaWVudCk7DQo+ICsNCj4gK3N0YXRpYyB2b2lkDQo+ICtuZnM0X3NodXRkb3duX2RzX2Ns
aWVudHMoc3RydWN0IG5mc19jbGllbnQgKmNscCkNCj4gK3sNCj4gKwlzdHJ1Y3QgbmZzNF9kc19z
ZXJ2ZXIgKmRzczsNCj4gKwlMSVNUX0hFQUQoc2h1dGRvd25fbGlzdCk7DQo+ICsNCj4gKwlzcGlu
X2xvY2soJmNscC0+Y2xfbG9jayk7DQo+ICsJd2hpbGUgKCFsaXN0X2VtcHR5KCZjbHAtPmNsX2Rz
X2NsaWVudHMpKSB7DQo+ICsJCWRzcyA9IGxpc3RfZW50cnkoY2xwLT5jbF9kc19jbGllbnRzLm5l
eHQsDQo+ICsJCQkJICBzdHJ1Y3QgbmZzNF9kc19zZXJ2ZXIsIGxpc3QpOw0KPiArCQlsaXN0X21v
dmUoJmRzcy0+bGlzdCwgJnNodXRkb3duX2xpc3QpOw0KDQpJcyB0aGlzIHN0ZXAgbmVjZXNzYXJ5
PyBXZSBrbm93IHRoYXQgbm9ib2R5IG90aGVyIHRoYW4gdXMgaXMgcmVmZXJlbmNpbmcNCnRoZSBu
ZnNfY2xpZW50IGF0IHRoaXMgcG9pbnQuDQoNCj4gKwl9DQo+ICsJc3Bpbl91bmxvY2soJmNscC0+
Y2xfbG9jayk7DQo+ICsNCj4gKwl3aGlsZSAoIWxpc3RfZW1wdHkoJnNodXRkb3duX2xpc3QpKSB7
DQo+ICsJCWRzcyA9IGxpc3RfZW50cnkoc2h1dGRvd25fbGlzdC5uZXh0LA0KPiArCQkJCSAgc3Ry
dWN0IG5mczRfZHNfc2VydmVyLCBsaXN0KTsNCj4gKwkJbGlzdF9kZWwoJmRzcy0+bGlzdCk7DQo+
ICsJCXJwY19zaHV0ZG93bl9jbGllbnQoZHNzLT5ycGNfY2xudCk7DQo+ICsJCWtmcmVlIChkc3Mp
Ow0KPiArCX0NCj4gK30NCj4gKw0KPiAgI2Vsc2UgLyogQ09ORklHX05GU19WNF8xICovDQo+ICBz
dGF0aWMgdm9pZCBuZnM0X3NodXRkb3duX3Nlc3Npb24oc3RydWN0IG5mc19jbGllbnQgKmNscCkN
Cj4gIHsNCj4gIH0NCj4gKw0KPiArc3RhdGljIHZvaWQNCj4gK25mczRfc2h1dGRvd25fZHNfY2xp
ZW50cyhzdHJ1Y3QgbmZzX2NsaWVudCAqY2xwKQ0KPiArew0KPiArfQ0KPiAgI2VuZGlmIC8qIENP
TkZJR19ORlNfVjRfMSAqLw0KPiAgDQo+ICBzdHJ1Y3QgbmZzX2NsaWVudCAqbmZzNF9hbGxvY19j
bGllbnQoY29uc3Qgc3RydWN0IG5mc19jbGllbnRfaW5pdGRhdGEgKmNsX2luaXQpDQo+IEBAIC03
Myw2ICsxODEsNyBAQCBzdHJ1Y3QgbmZzX2NsaWVudCAqbmZzNF9hbGxvY19jbGllbnQoY29uc3Qg
c3RydWN0IG5mc19jbGllbnRfaW5pdGRhdGEgKmNsX2luaXQpDQo+ICANCj4gIAlzcGluX2xvY2tf
aW5pdCgmY2xwLT5jbF9sb2NrKTsNCj4gIAlJTklUX0RFTEFZRURfV09SSygmY2xwLT5jbF9yZW5l
d2QsIG5mczRfcmVuZXdfc3RhdGUpOw0KPiArCUlOSVRfTElTVF9IRUFEKCZjbHAtPmNsX2RzX2Ns
aWVudHMpOw0KPiAgCXJwY19pbml0X3dhaXRfcXVldWUoJmNscC0+Y2xfcnBjd2FpdHEsICJORlMg
Y2xpZW50Iik7DQo+ICAJY2xwLT5jbF9zdGF0ZSA9IDEgPDwgTkZTNENMTlRfTEVBU0VfRVhQSVJF
RDsNCj4gIAljbHAtPmNsX21pbm9ydmVyc2lvbiA9IGNsX2luaXQtPm1pbm9ydmVyc2lvbjsNCj4g
QEAgLTk3LDYgKzIwNiw3IEBAIHN0YXRpYyB2b2lkIG5mczRfc2h1dGRvd25fY2xpZW50KHN0cnVj
dCBuZnNfY2xpZW50ICpjbHApDQo+ICB7DQo+ICAJaWYgKF9fdGVzdF9hbmRfY2xlYXJfYml0KE5G
U19DU19SRU5FV0QsICZjbHAtPmNsX3Jlc19zdGF0ZSkpDQo+ICAJCW5mczRfa2lsbF9yZW5ld2Qo
Y2xwKTsNCj4gKwluZnM0X3NodXRkb3duX2RzX2NsaWVudHMoY2xwKTsNCj4gIAluZnM0X3NodXRk
b3duX3Nlc3Npb24oY2xwKTsNCj4gIAluZnM0X2Rlc3Ryb3lfY2FsbGJhY2soY2xwKTsNCj4gIAlp
ZiAoX190ZXN0X2FuZF9jbGVhcl9iaXQoTkZTX0NTX0lETUFQLCAmY2xwLT5jbF9yZXNfc3RhdGUp
KQ0KPiBkaWZmIC0tZ2l0IGEvZnMvbmZzL25mczRmaWxlbGF5b3V0LmMgYi9mcy9uZnMvbmZzNGZp
bGVsYXlvdXQuYw0KPiBpbmRleCBhNzBjYjNhLi5iODY0NjRiIDEwMDY0NA0KPiAtLS0gYS9mcy9u
ZnMvbmZzNGZpbGVsYXlvdXQuYw0KPiArKysgYi9mcy9uZnMvbmZzNGZpbGVsYXlvdXQuYw0KPiBA
QCAtNTI4LDYgKzUyOCw3IEBAIGZpbGVsYXlvdXRfcmVhZF9wYWdlbGlzdChzdHJ1Y3QgbmZzX3Jl
YWRfZGF0YSAqZGF0YSkNCj4gIAlzdHJ1Y3QgbmZzX3BnaW9faGVhZGVyICpoZHIgPSBkYXRhLT5o
ZWFkZXI7DQo+ICAJc3RydWN0IHBuZnNfbGF5b3V0X3NlZ21lbnQgKmxzZWcgPSBoZHItPmxzZWc7
DQo+ICAJc3RydWN0IG5mczRfcG5mc19kcyAqZHM7DQo+ICsJc3RydWN0IHJwY19jbG50ICpkc19j
bG50Ow0KPiAgCWxvZmZfdCBvZmZzZXQgPSBkYXRhLT5hcmdzLm9mZnNldDsNCj4gIAl1MzIgaiwg
aWR4Ow0KPiAgCXN0cnVjdCBuZnNfZmggKmZoOw0KPiBAQCAtNTQyLDYgKzU0MywxMSBAQCBmaWxl
bGF5b3V0X3JlYWRfcGFnZWxpc3Qoc3RydWN0IG5mc19yZWFkX2RhdGEgKmRhdGEpDQo+ICAJZHMg
PSBuZnM0X2ZsX3ByZXBhcmVfZHMobHNlZywgaWR4KTsNCj4gIAlpZiAoIWRzKQ0KPiAgCQlyZXR1
cm4gUE5GU19OT1RfQVRURU1QVEVEOw0KPiArDQo+ICsJZHNfY2xudCA9IG5mczRfZmluZF9vcl9j
cmVhdGVfZHNfY2xpZW50KGRzLT5kc19jbHAsIGhkci0+aW5vZGUpOw0KPiArCWlmIChJU19FUlIo
ZHNfY2xudCkpDQo+ICsJCXJldHVybiBQTkZTX05PVF9BVFRFTVBURUQ7DQo+ICsNCj4gIAlkcHJp
bnRrKCIlcyBVU0UgRFM6ICVzIGNsX2NvdW50ICVkXG4iLCBfX2Z1bmNfXywNCj4gIAkJZHMtPmRz
X3JlbW90ZXN0ciwgYXRvbWljX3JlYWQoJmRzLT5kc19jbHAtPmNsX2NvdW50KSk7DQo+ICANCj4g
QEAgLTU1Niw3ICs1NjIsNyBAQCBmaWxlbGF5b3V0X3JlYWRfcGFnZWxpc3Qoc3RydWN0IG5mc19y
ZWFkX2RhdGEgKmRhdGEpDQo+ICAJZGF0YS0+bWRzX29mZnNldCA9IG9mZnNldDsNCj4gIA0KPiAg
CS8qIFBlcmZvcm0gYW4gYXN5bmNocm9ub3VzIHJlYWQgdG8gZHMgKi8NCj4gLQluZnNfaW5pdGlh
dGVfcmVhZChkcy0+ZHNfY2xwLT5jbF9ycGNjbGllbnQsIGRhdGEsDQo+ICsJbmZzX2luaXRpYXRl
X3JlYWQoZHNfY2xudCwgZGF0YSwNCj4gIAkJCQkgICZmaWxlbGF5b3V0X3JlYWRfY2FsbF9vcHMs
IFJQQ19UQVNLX1NPRlRDT05OKTsNCj4gIAlyZXR1cm4gUE5GU19BVFRFTVBURUQ7DQo+ICB9DQo+
IEBAIC01NjgsNiArNTc0LDcgQEAgZmlsZWxheW91dF93cml0ZV9wYWdlbGlzdChzdHJ1Y3QgbmZz
X3dyaXRlX2RhdGEgKmRhdGEsIGludCBzeW5jKQ0KPiAgCXN0cnVjdCBuZnNfcGdpb19oZWFkZXIg
KmhkciA9IGRhdGEtPmhlYWRlcjsNCj4gIAlzdHJ1Y3QgcG5mc19sYXlvdXRfc2VnbWVudCAqbHNl
ZyA9IGhkci0+bHNlZzsNCj4gIAlzdHJ1Y3QgbmZzNF9wbmZzX2RzICpkczsNCj4gKwlzdHJ1Y3Qg
cnBjX2NsbnQgKmRzX2NsbnQ7DQo+ICAJbG9mZl90IG9mZnNldCA9IGRhdGEtPmFyZ3Mub2Zmc2V0
Ow0KPiAgCXUzMiBqLCBpZHg7DQo+ICAJc3RydWN0IG5mc19maCAqZmg7DQo+IEBAIC01NzgsNiAr
NTg1LDExIEBAIGZpbGVsYXlvdXRfd3JpdGVfcGFnZWxpc3Qoc3RydWN0IG5mc193cml0ZV9kYXRh
ICpkYXRhLCBpbnQgc3luYykNCj4gIAlkcyA9IG5mczRfZmxfcHJlcGFyZV9kcyhsc2VnLCBpZHgp
Ow0KPiAgCWlmICghZHMpDQo+ICAJCXJldHVybiBQTkZTX05PVF9BVFRFTVBURUQ7DQo+ICsNCj4g
Kwlkc19jbG50ID0gbmZzNF9maW5kX29yX2NyZWF0ZV9kc19jbGllbnQoZHMtPmRzX2NscCwgaGRy
LT5pbm9kZSk7DQo+ICsJaWYgKElTX0VSUihkc19jbG50KSkNCj4gKwkJcmV0dXJuIFBORlNfTk9U
X0FUVEVNUFRFRDsNCj4gKw0KPiAgCWRwcmludGsoIiVzIGlubyAlbHUgc3luYyAlZCByZXEgJVp1
QCVsbHUgRFM6ICVzIGNsX2NvdW50ICVkXG4iLA0KPiAgCQlfX2Z1bmNfXywgaGRyLT5pbm9kZS0+
aV9pbm8sIHN5bmMsIChzaXplX3QpIGRhdGEtPmFyZ3MuY291bnQsDQo+ICAJCW9mZnNldCwgZHMt
PmRzX3JlbW90ZXN0ciwgYXRvbWljX3JlYWQoJmRzLT5kc19jbHAtPmNsX2NvdW50KSk7DQo+IEBA
IC01OTUsNyArNjA3LDcgQEAgZmlsZWxheW91dF93cml0ZV9wYWdlbGlzdChzdHJ1Y3QgbmZzX3dy
aXRlX2RhdGEgKmRhdGEsIGludCBzeW5jKQ0KPiAgCWRhdGEtPmFyZ3Mub2Zmc2V0ID0gZmlsZWxh
eW91dF9nZXRfZHNlcnZlcl9vZmZzZXQobHNlZywgb2Zmc2V0KTsNCj4gIA0KPiAgCS8qIFBlcmZv
cm0gYW4gYXN5bmNocm9ub3VzIHdyaXRlICovDQo+IC0JbmZzX2luaXRpYXRlX3dyaXRlKGRzLT5k
c19jbHAtPmNsX3JwY2NsaWVudCwgZGF0YSwNCj4gKwluZnNfaW5pdGlhdGVfd3JpdGUoZHNfY2xu
dCwgZGF0YSwNCj4gIAkJCQkgICAgJmZpbGVsYXlvdXRfd3JpdGVfY2FsbF9vcHMsIHN5bmMsDQo+
ICAJCQkJICAgIFJQQ19UQVNLX1NPRlRDT05OKTsNCj4gIAlyZXR1cm4gUE5GU19BVFRFTVBURUQ7
DQo+IEBAIC0xMTA1LDE2ICsxMTE3LDE5IEBAIHN0YXRpYyBpbnQgZmlsZWxheW91dF9pbml0aWF0
ZV9jb21taXQoc3RydWN0IG5mc19jb21taXRfZGF0YSAqZGF0YSwgaW50IGhvdykNCj4gIHsNCj4g
IAlzdHJ1Y3QgcG5mc19sYXlvdXRfc2VnbWVudCAqbHNlZyA9IGRhdGEtPmxzZWc7DQo+ICAJc3Ry
dWN0IG5mczRfcG5mc19kcyAqZHM7DQo+ICsJc3RydWN0IHJwY19jbG50ICpkc19jbG50Ow0KPiAg
CXUzMiBpZHg7DQo+ICAJc3RydWN0IG5mc19maCAqZmg7DQo+ICANCj4gIAlpZHggPSBjYWxjX2Rz
X2luZGV4X2Zyb21fY29tbWl0KGxzZWcsIGRhdGEtPmRzX2NvbW1pdF9pbmRleCk7DQo+ICAJZHMg
PSBuZnM0X2ZsX3ByZXBhcmVfZHMobHNlZywgaWR4KTsNCj4gLQlpZiAoIWRzKSB7DQo+IC0JCXBy
ZXBhcmVfdG9fcmVzZW5kX3dyaXRlcyhkYXRhKTsNCj4gLQkJZmlsZWxheW91dF9jb21taXRfcmVs
ZWFzZShkYXRhKTsNCj4gLQkJcmV0dXJuIC1FQUdBSU47DQo+IC0JfQ0KPiArCWlmICghZHMpDQo+
ICsJCWdvdG8gb3V0X2VycjsNCj4gKw0KPiArCWRzX2NsbnQgPSBuZnM0X2ZpbmRfb3JfY3JlYXRl
X2RzX2NsaWVudChkcy0+ZHNfY2xwLCBkYXRhLT5pbm9kZSk7DQo+ICsJaWYgKElTX0VSUihkc19j
bG50KSkNCj4gKwkJZ290byBvdXRfZXJyOw0KPiArDQo+ICAJZHByaW50aygiJXMgaW5vICVsdSwg
aG93ICVkIGNsX2NvdW50ICVkXG4iLCBfX2Z1bmNfXywNCj4gIAkJZGF0YS0+aW5vZGUtPmlfaW5v
LCBob3csIGF0b21pY19yZWFkKCZkcy0+ZHNfY2xwLT5jbF9jb3VudCkpOw0KPiAgCWRhdGEtPmNv
bW1pdF9kb25lX2NiID0gZmlsZWxheW91dF9jb21taXRfZG9uZV9jYjsNCj4gQEAgLTExMjMsOSAr
MTEzOCwxMyBAQCBzdGF0aWMgaW50IGZpbGVsYXlvdXRfaW5pdGlhdGVfY29tbWl0KHN0cnVjdCBu
ZnNfY29tbWl0X2RhdGEgKmRhdGEsIGludCBob3cpDQo+ICAJZmggPSBzZWxlY3RfZHNfZmhfZnJv
bV9jb21taXQobHNlZywgZGF0YS0+ZHNfY29tbWl0X2luZGV4KTsNCj4gIAlpZiAoZmgpDQo+ICAJ
CWRhdGEtPmFyZ3MuZmggPSBmaDsNCj4gLQlyZXR1cm4gbmZzX2luaXRpYXRlX2NvbW1pdChkcy0+
ZHNfY2xwLT5jbF9ycGNjbGllbnQsIGRhdGEsDQo+ICsJcmV0dXJuIG5mc19pbml0aWF0ZV9jb21t
aXQoZHNfY2xudCwgZGF0YSwNCj4gIAkJCQkgICAmZmlsZWxheW91dF9jb21taXRfY2FsbF9vcHMs
IGhvdywNCj4gIAkJCQkgICBSUENfVEFTS19TT0ZUQ09OTik7DQo+ICtvdXRfZXJyOg0KPiArCXBy
ZXBhcmVfdG9fcmVzZW5kX3dyaXRlcyhkYXRhKTsNCj4gKwlmaWxlbGF5b3V0X2NvbW1pdF9yZWxl
YXNlKGRhdGEpOw0KPiArCXJldHVybiAtRUFHQUlOOw0KPiAgfQ0KPiAgDQo+ICBzdGF0aWMgaW50
DQo+IGRpZmYgLS1naXQgYS9pbmNsdWRlL2xpbnV4L25mc19mc19zYi5oIGIvaW5jbHVkZS9saW51
eC9uZnNfZnNfc2IuaA0KPiBpbmRleCBkMjIxMjQzLi40ZDQ3NmU3IDEwMDY0NA0KPiAtLS0gYS9p
bmNsdWRlL2xpbnV4L25mc19mc19zYi5oDQo+ICsrKyBiL2luY2x1ZGUvbGludXgvbmZzX2ZzX3Ni
LmgNCj4gQEAgLTU2LDYgKzU2LDcgQEAgc3RydWN0IG5mc19jbGllbnQgew0KPiAgCXN0cnVjdCBy
cGNfY3JlZAkJKmNsX21hY2hpbmVfY3JlZDsNCj4gIA0KPiAgI2lmIElTX0VOQUJMRUQoQ09ORklH
X05GU19WNCkNCj4gKwlzdHJ1Y3QgbGlzdF9oZWFkCWNsX2RzX2NsaWVudHM7IC8qIGF1dGggZmxh
dm9yIGRhdGEgc2VydmVycyAqLw0KPiAgCXU2NAkJCWNsX2NsaWVudGlkOwkvKiBjb25zdGFudCAq
Lw0KPiAgCW5mczRfdmVyaWZpZXIJCWNsX2NvbmZpcm07CS8qIENsaWVudGlkIHZlcmlmaWVyICov
DQo+ICAJdW5zaWduZWQgbG9uZwkJY2xfc3RhdGU7DQoNCi0tIA0KVHJvbmQgTXlrbGVidXN0DQpM
aW51eCBORlMgY2xpZW50IG1haW50YWluZXINCg0KTmV0QXBwDQpUcm9uZC5NeWtsZWJ1c3RAbmV0
YXBwLmNvbQ0Kd3d3Lm5ldGFwcC5jb20NCg==
On Sep 5, 2013, at 1:48 PM, "Myklebust, Trond" <[email protected]>
wrote:
> On Thu, 2013-09-05 at 13:30 -0400, [email protected] wrote:
>> From: Andy Adamson <[email protected]>
>>
>> Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible"
>> uses the nfs_client cl_rpcclient for all state management operations, and
>> will use krb5i or auth_sys with no regard to the mount command authflavor
>> choice.
>>
>> The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all
>> non-state management operations with a different nfs_server for each fsid
>> encountered traversing the mount point, each with a potentially different
>> auth flavor.
>>
>> pNFS data servers are not mounted in the normal sense as there is no associated
>> nfs_server structure. Data servers can also export multiple fsids, each with
>> a potentially different auth flavor.
>>
>> Data servers need to use the same authflavor as the MDS server rpc client for
>> non-state management operations. Populate a list of rpc clients with the MDS
>> server rpc client auth flavor for the DS to use.
>>
>> Signed-off-by: Andy Adamson <[email protected]>
>> ---
>> fs/nfs/internal.h | 2 +
>> fs/nfs/nfs4client.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++
>> fs/nfs/nfs4filelayout.c | 35 +++++++++++----
>> include/linux/nfs_fs_sb.h | 1 +
>> 4 files changed, 140 insertions(+), 8 deletions(-)
>
> That looks a lot better. See comments inline below.
>
>> diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
>> index 2415198..23ec6e8 100644
>> --- a/fs/nfs/internal.h
>> +++ b/fs/nfs/internal.h
>> @@ -186,6 +186,8 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
>> int ds_addrlen, int ds_proto,
>> unsigned int ds_timeo,
>> unsigned int ds_retrans);
>> +extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
>> + struct inode *);
>> #ifdef CONFIG_PROC_FS
>> extern int __init nfs_fs_proc_init(void);
>> extern void nfs_fs_proc_exit(void);
>> diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
>> index 767a5e3..a8dd396 100644
>> --- a/fs/nfs/nfs4client.c
>> +++ b/fs/nfs/nfs4client.c
>> @@ -49,10 +49,118 @@ static void nfs4_shutdown_session(struct nfs_client *clp)
>> }
>>
>> }
>> +
>> +/**
>> + * Per auth flavor data server rpc clients
>> + */
>> +struct nfs4_ds_server {
>> + struct list_head list; /* ds_clp->cl_ds_clients */
>> + struct rpc_clnt *rpc_clnt;
>> +};
>> +
>> +static struct nfs4_ds_server *
>> +nfs4_find_or_add_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor,
>> + struct nfs4_ds_server *new)
>> +{
>> + struct nfs4_ds_server *dss, *tmp;
>> +
>> + spin_lock(&ds_clp->cl_lock);
>> + list_for_each_entry_safe(dss, tmp, &ds_clp->cl_ds_clients, list) {
>
> Nit: why do we need list_for_each_entry_safe() here? You're not removing
> anything from that list.
>
>> + if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
>> + continue;
>> + goto out;
>> + }
>> + if (new)
>> + list_add(&new->list, &ds_clp->cl_ds_clients);
>> + dss = new;
>> +out:
>> + spin_unlock(&ds_clp->cl_lock); /* need some lock to protect list */
>> + return dss;
>> +}
>> +
>> +static struct nfs4_ds_server *
>> +nfs4_alloc_ds_server(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
>> +{
>> + struct nfs4_ds_server *dss;
>> +
>> + dss = kmalloc(sizeof(*dss), GFP_NOFS);
>> + if (dss == NULL)
>> + return ERR_PTR(-ENOMEM);
>> +
>> + dss->rpc_clnt = rpc_clone_client_set_auth(ds_clp->cl_rpcclient, flavor);
>> + if (IS_ERR(dss->rpc_clnt)) {
>> + int err = PTR_ERR(dss->rpc_clnt);
>> + kfree (dss);
>> + return ERR_PTR(err);
>> + }
>> + INIT_LIST_HEAD(&dss->list);
>> +
>> + return dss;
>> +}
>> +
>> +static void
>> +nfs4_free_ds_server(struct nfs4_ds_server *dss)
>> +{
>> + rpc_release_client(dss->rpc_clnt);
>> + kfree(dss);
>> +}
>> +
>> +/**
>> + * Find or create a DS rpc client with th MDS server rpc client auth flavor
>> + * in the nfs_client cl_ds_clients list.
>> + */
>> +struct rpc_clnt *
>> +nfs4_find_or_create_ds_client(struct nfs_client *ds_clp, struct inode *inode)
>> +{
>> + struct nfs4_ds_server *dss, *new;
>> + rpc_authflavor_t flavor = NFS_SERVER(inode)->client->cl_auth->au_flavor;
>> +
>> + dss = nfs4_find_or_add_ds_client(ds_clp, flavor, NULL);
>
> Does it make sense to perhaps use RCU in this case (but not the one
> below) in order to avoid the contention on clp->cl_lock?
OK - since this is in the I/O path, I can do that.
>
>> + if (dss != NULL)
>> + goto out;
>> + new = nfs4_alloc_ds_server(ds_clp, flavor);
>> + if (IS_ERR(new))
>> + return (struct rpc_clnt *)new;
>
> Nit: you should use ERR_CAST() rather than an explicit cast above
>
>> + dss = nfs4_find_or_add_ds_client(ds_clp, flavor, new);
>> + if (dss != new)
>> + nfs4_free_ds_server(new);
>> +out:
>> + return dss->rpc_clnt;
>> +}
>> +EXPORT_SYMBOL_GPL(nfs4_find_or_create_ds_client);
>> +
>> +static void
>> +nfs4_shutdown_ds_clients(struct nfs_client *clp)
>> +{
>> + struct nfs4_ds_server *dss;
>> + LIST_HEAD(shutdown_list);
>> +
>> + spin_lock(&clp->cl_lock);
>> + while (!list_empty(&clp->cl_ds_clients)) {
>> + dss = list_entry(clp->cl_ds_clients.next,
>> + struct nfs4_ds_server, list);
>> + list_move(&dss->list, &shutdown_list);
>
> Is this step necessary? We know that nobody other than us is referencing
> the nfs_client at this point.
Because of the kfree under the spin lock - but perhaps that is no longer an issue?
-->Andy
>
>> + }
>> + spin_unlock(&clp->cl_lock);
>> +
>> + while (!list_empty(&shutdown_list)) {
>> + dss = list_entry(shutdown_list.next,
>> + struct nfs4_ds_server, list);
>> + list_del(&dss->list);
>> + rpc_shutdown_client(dss->rpc_clnt);
>> + kfree (dss);
>> + }
>> +}
>> +
>> #else /* CONFIG_NFS_V4_1 */
>> static void nfs4_shutdown_session(struct nfs_client *clp)
>> {
>> }
>> +
>> +static void
>> +nfs4_shutdown_ds_clients(struct nfs_client *clp)
>> +{
>> +}
>> #endif /* CONFIG_NFS_V4_1 */
>>
>> struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
>> @@ -73,6 +181,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
>>
>> spin_lock_init(&clp->cl_lock);
>> INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
>> + INIT_LIST_HEAD(&clp->cl_ds_clients);
>> rpc_init_wait_queue(&clp->cl_rpcwaitq, "NFS client");
>> clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED;
>> clp->cl_minorversion = cl_init->minorversion;
>> @@ -97,6 +206,7 @@ static void nfs4_shutdown_client(struct nfs_client *clp)
>> {
>> if (__test_and_clear_bit(NFS_CS_RENEWD, &clp->cl_res_state))
>> nfs4_kill_renewd(clp);
>> + nfs4_shutdown_ds_clients(clp);
>> nfs4_shutdown_session(clp);
>> nfs4_destroy_callback(clp);
>> if (__test_and_clear_bit(NFS_CS_IDMAP, &clp->cl_res_state))
>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
>> index a70cb3a..b86464b 100644
>> --- a/fs/nfs/nfs4filelayout.c
>> +++ b/fs/nfs/nfs4filelayout.c
>> @@ -528,6 +528,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
>> struct nfs_pgio_header *hdr = data->header;
>> struct pnfs_layout_segment *lseg = hdr->lseg;
>> struct nfs4_pnfs_ds *ds;
>> + struct rpc_clnt *ds_clnt;
>> loff_t offset = data->args.offset;
>> u32 j, idx;
>> struct nfs_fh *fh;
>> @@ -542,6 +543,11 @@ filelayout_read_pagelist(struct nfs_read_data *data)
>> ds = nfs4_fl_prepare_ds(lseg, idx);
>> if (!ds)
>> return PNFS_NOT_ATTEMPTED;
>> +
>> + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
>> + if (IS_ERR(ds_clnt))
>> + return PNFS_NOT_ATTEMPTED;
>> +
>> dprintk("%s USE DS: %s cl_count %d\n", __func__,
>> ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
>>
>> @@ -556,7 +562,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
>> data->mds_offset = offset;
>>
>> /* Perform an asynchronous read to ds */
>> - nfs_initiate_read(ds->ds_clp->cl_rpcclient, data,
>> + nfs_initiate_read(ds_clnt, data,
>> &filelayout_read_call_ops, RPC_TASK_SOFTCONN);
>> return PNFS_ATTEMPTED;
>> }
>> @@ -568,6 +574,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
>> struct nfs_pgio_header *hdr = data->header;
>> struct pnfs_layout_segment *lseg = hdr->lseg;
>> struct nfs4_pnfs_ds *ds;
>> + struct rpc_clnt *ds_clnt;
>> loff_t offset = data->args.offset;
>> u32 j, idx;
>> struct nfs_fh *fh;
>> @@ -578,6 +585,11 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
>> ds = nfs4_fl_prepare_ds(lseg, idx);
>> if (!ds)
>> return PNFS_NOT_ATTEMPTED;
>> +
>> + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
>> + if (IS_ERR(ds_clnt))
>> + return PNFS_NOT_ATTEMPTED;
>> +
>> dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d\n",
>> __func__, hdr->inode->i_ino, sync, (size_t) data->args.count,
>> offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
>> @@ -595,7 +607,7 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
>> data->args.offset = filelayout_get_dserver_offset(lseg, offset);
>>
>> /* Perform an asynchronous write */
>> - nfs_initiate_write(ds->ds_clp->cl_rpcclient, data,
>> + nfs_initiate_write(ds_clnt, data,
>> &filelayout_write_call_ops, sync,
>> RPC_TASK_SOFTCONN);
>> return PNFS_ATTEMPTED;
>> @@ -1105,16 +1117,19 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
>> {
>> struct pnfs_layout_segment *lseg = data->lseg;
>> struct nfs4_pnfs_ds *ds;
>> + struct rpc_clnt *ds_clnt;
>> u32 idx;
>> struct nfs_fh *fh;
>>
>> idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
>> ds = nfs4_fl_prepare_ds(lseg, idx);
>> - if (!ds) {
>> - prepare_to_resend_writes(data);
>> - filelayout_commit_release(data);
>> - return -EAGAIN;
>> - }
>> + if (!ds)
>> + goto out_err;
>> +
>> + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode);
>> + if (IS_ERR(ds_clnt))
>> + goto out_err;
>> +
>> dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
>> data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count));
>> data->commit_done_cb = filelayout_commit_done_cb;
>> @@ -1123,9 +1138,13 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
>> fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
>> if (fh)
>> data->args.fh = fh;
>> - return nfs_initiate_commit(ds->ds_clp->cl_rpcclient, data,
>> + return nfs_initiate_commit(ds_clnt, data,
>> &filelayout_commit_call_ops, how,
>> RPC_TASK_SOFTCONN);
>> +out_err:
>> + prepare_to_resend_writes(data);
>> + filelayout_commit_release(data);
>> + return -EAGAIN;
>> }
>>
>> static int
>> diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
>> index d221243..4d476e7 100644
>> --- a/include/linux/nfs_fs_sb.h
>> +++ b/include/linux/nfs_fs_sb.h
>> @@ -56,6 +56,7 @@ struct nfs_client {
>> struct rpc_cred *cl_machine_cred;
>>
>> #if IS_ENABLED(CONFIG_NFS_V4)
>> + struct list_head cl_ds_clients; /* auth flavor data servers */
>> u64 cl_clientid; /* constant */
>> nfs4_verifier cl_confirm; /* Clientid verifier */
>> unsigned long cl_state;
>
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> [email protected]
> http://www.netapp.com