craa_type_mask is bitmap4 according to RFC5661. We need to expect a length before
extracting its value.
Signed-off-by: Peng Tao <[email protected]>
---
Resend this one as it is a dependency for "[PATCH 2/2] nfs41: handle BLK_LAYOUT CB_RECALL_ANY".
fs/nfs/callback_xdr.c | 11 ++++++-----
1 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 918ad64..ee1a5b3 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -488,17 +488,18 @@ static __be32 decode_recallany_args(struct svc_rqst *rqstp,
struct xdr_stream *xdr,
struct cb_recallanyargs *args)
{
- __be32 *p;
+ uint32_t bitmap[2];
+ __be32 *p, status;
args->craa_addr = svc_addr(rqstp);
p = read_buf(xdr, 4);
if (unlikely(p == NULL))
return htonl(NFS4ERR_BADXDR);
args->craa_objs_to_keep = ntohl(*p++);
- p = read_buf(xdr, 4);
- if (unlikely(p == NULL))
- return htonl(NFS4ERR_BADXDR);
- args->craa_type_mask = ntohl(*p);
+ status = decode_bitmap(xdr, bitmap);
+ if (unlikely(status))
+ return status;
+ args->craa_type_mask = bitmap[0];
return 0;
}
--
1.7.1.262.g5ef3d
For blocklayout, we need to issue layoutreturn to return layouts when
handling CB_RECALL_ANY.
Signed-off-by: Peng Tao <[email protected]>
---
fs/nfs/callback_proc.c | 55 +++++++++++++++++++++++++++++++++++++++++-------
include/linux/nfs4.h | 3 +-
2 files changed, 49 insertions(+), 9 deletions(-)
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 43926ad..11f79c2 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -160,7 +160,8 @@ static u32 initiate_file_draining(struct nfs_client *clp,
}
static u32 initiate_bulk_draining(struct nfs_client *clp,
- struct cb_layoutrecallargs *args)
+ struct cb_layoutrecallargs *args,
+ int sendreturn)
{
struct nfs_server *server;
struct pnfs_layout_hdr *lo;
@@ -204,12 +205,47 @@ static u32 initiate_bulk_draining(struct nfs_client *clp,
list_del_init(&lo->plh_bulk_recall);
spin_unlock(&ino->i_lock);
pnfs_free_lseg_list(&free_me_list);
+ if (sendreturn && list_empty(&lo->plh_segs))
+ pnfs_return_layout(ino);
put_layout_hdr(lo);
iput(ino);
}
return rv;
}
+struct recallany_data {
+ struct nfs_client *clp;
+ struct work_struct ra_work;
+};
+
+static void layout_recallany_draining(struct work_struct *work)
+{
+ struct recallany_data *ra;
+ struct cb_layoutrecallargs args;
+
+ memset(&args, 0, sizeof(args));
+ ra = container_of(work, struct recallany_data, ra_work);
+ /* Ignore draining error. Per RFC, if layoutreturns are not sent, it is up
+ * to server to handle the situation (e.g., send specific layoutrecalls).
+ */
+ initiate_bulk_draining(ra->clp, &args, 1);
+ kfree(ra);
+}
+
+static u32 init_recallany_draining(struct nfs_client *clp)
+{
+ struct recallany_data *ra;
+
+ ra = kmalloc(sizeof(*ra), GFP_NOFS);
+ if (ra) {
+ ra->clp = clp;
+ INIT_WORK(&ra->ra_work, layout_recallany_draining);
+ schedule_work(&ra->ra_work);
+ return NFS4_OK;
+ }
+ return NFS4ERR_DELAY;
+}
+
static u32 do_callback_layoutrecall(struct nfs_client *clp,
struct cb_layoutrecallargs *args)
{
@@ -220,8 +256,10 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,
goto out;
if (args->cbl_recall_type == RETURN_FILE)
res = initiate_file_draining(clp, args);
+ if (args->cbl_recall_type == RETURN_ANY)
+ res = init_recallany_draining(clp);
else
- res = initiate_bulk_draining(clp, args);
+ res = initiate_bulk_draining(clp, args, 0);
clear_bit(NFS4CLNT_LAYOUTRECALL, &clp->cl_state);
out:
dprintk("%s returning %i\n", __func__, res);
@@ -245,15 +283,13 @@ __be32 nfs4_callback_layoutrecall(struct cb_layoutrecallargs *args,
return cpu_to_be32(res);
}
-static void pnfs_recall_all_layouts(struct nfs_client *clp)
+static __be32 pnfs_recall_all_layouts(struct nfs_client *clp, uint32_t type)
{
struct cb_layoutrecallargs args;
- /* Pretend we got a CB_LAYOUTRECALL(ALL) */
memset(&args, 0, sizeof(args));
- args.cbl_recall_type = RETURN_ALL;
- /* FIXME we ignore errors, what should we do? */
- do_callback_layoutrecall(clp, &args);
+ args.cbl_recall_type = type;
+ return cpu_to_be32(do_callback_layoutrecall(clp, &args));
}
__be32 nfs4_callback_devicenotify(struct cb_devicenotifyargs *args,
@@ -533,7 +569,10 @@ __be32 nfs4_callback_recallany(struct cb_recallanyargs *args, void *dummy,
flags |= FMODE_WRITE;
if (test_bit(RCA4_TYPE_MASK_FILE_LAYOUT, (const unsigned long *)
&args->craa_type_mask))
- pnfs_recall_all_layouts(cps->clp);
+ pnfs_recall_all_layouts(cps->clp, RETURN_ALL);
+ if (test_bit(RCA4_TYPE_MASK_BLK_LAYOUT, (const unsigned long *)
+ &args->craa_type_mask))
+ status = pnfs_recall_all_layouts(cps->clp, RETURN_ANY);
if (flags)
nfs_expire_all_delegation_types(cps->clp, flags);
out:
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 76f99e8..1f71a9e 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -597,7 +597,8 @@ enum pnfs_layouttype {
enum pnfs_layoutreturn_type {
RETURN_FILE = 1,
RETURN_FSID = 2,
- RETURN_ALL = 3
+ RETURN_ALL = 3,
+ RETURN_ANY = 4
};
enum pnfs_iomode {
--
1.7.1.262.g5ef3d
On Tue, Nov 1, 2011 at 12:45 AM, Trond Myklebust
<[email protected]> wrote:
> On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
>> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
>> <[email protected]> wrote:
>> > On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
>> >> For blocklayout, we need to issue layoutreturn to return layouts when
>> >> handling CB_RECALL_ANY.
>> >
>> > Why?
>> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
>> that server wants client to return layout. And server will be waiting
>> for layoutreturn in such case.
>
> No it doesn't. NFS4_OK means that the client acknowledges that it has
> been given a new limit on the number of recallable objects it can keep.
> There is no requirement in the text that it should send layoutreturn or
> that the server should expect that.
Per RFC5661 section 20.6.3:
It is the job of the client to bring down the size of the
recallable object set in line with each CB_RECALL_ANY received
For layout objects, the only way client can bring down the object
number is to send layoutreturn.
Besides, the moment client gets CB_RECALL_ANY is likely because server
reaches some resource threshold. If client doesn't send layoutreturn,
server can refusing client from getting more layouts.
>
> In any case, there is no reason to make a difference between block,
> object and file layouts when it comes to CB_RECALL_ANY. The code to
> handle it should be the same for all.
>
object layout isn't handling CB_RECALL_ANY yet. And from current code,
I was assuming file layouts have their own method at server side to
handle it (e.g., translating NFS4_OK of CB_RECALL_ANY into client has
dropped all its layout...).So I made it for block only.
Thanks,
Tao
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> [email protected]
> http://www.netapp.com
>
>
--
Thanks,
-Bergwolf
CB_RECALL_ANY was added based on Panasas experience with its own protocols. When you have
100's (or 1000's) of clients and millions of files for which you maintain state, it is
very much worth optimizing how the server can reclaim this state.
Brent
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Jim Rees
Sent: Monday, October 31, 2011 11:32 AM
To: Trond Myklebust
Cc: [email protected]; nfsv4 list
Subject: Re: [nfsv4] [PATCH 2/2] nfs41: handle BLK_LAYOUT CB_RECALL_ANY
Trond Myklebust wrote:
I don't necessarily disagree with what you are saying, but I have yet to
see a single server side implementation of CB_RECALL_ANY, let alone any
numbers that indicate performance or responsiveness problems resulting
from our existing client-side implementation.
I therefore find it hard to understand why optimising this particular
code is such a high priority, or why a patch that is adding per-file
layoutreturns to initiate_bulk_draining() is going to help anything at
all.
Testing between the linux block layout client and the EMC block layout
server revealed a deadlock when the server had handed out some number of
layouts and couldn't hand out any more. So now the EMC block layout server
implements CB_RECALL_ANY. So yes, this solves a real world problem, and
yes, there is a server that implements this.
We had some discussions at the time, and I don't remember if those were on
the linux-nfs list or in some other forum. We decided that the client was
in the best position to decide which layouts were no longer needed, so we
needed some way for the server to tell the client to return some layouts
without specifying which ones. CB_RECALL_ANY seemed custom-made for this
purpose, so we used it.
I don't think it would be appropriate for the server to recall all layouts
when it only needs some of them back.
_______________________________________________
nfsv4 mailing list
[email protected]
https://www.ietf.org/mailman/listinfo/nfsv4
On 2011-10-31 18:45, Trond Myklebust wrote:
> On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
>> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
>> <[email protected]> wrote:
>>> On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
>>>> For blocklayout, we need to issue layoutreturn to return layouts when
>>>> handling CB_RECALL_ANY.
>>>
>>> Why?
>> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
>> that server wants client to return layout. And server will be waiting
>> for layoutreturn in such case.
>
> No it doesn't. NFS4_OK means that the client acknowledges that it has
> been given a new limit on the number of recallable objects it can keep.
> There is no requirement in the text that it should send layoutreturn or
> that the server should expect that.
The motivation for CB_RECALL_ANY is to reduce the state on the *server* side.
Quoting from RFC5661:
The server may decide that it cannot hold all of the state for
recallable objects, such as delegations and layouts, without running
out of resources. In such a case, while not optimal, the server is
free to recall individual objects to reduce the load.
...
In order to implement an effective reclaim scheme for such objects,
the server's knowledge of available resources must be used to
determine when objects must be recalled with the clients selecting
the actual objects to be returned.
^^^^^^^^^^^^^^
...
When a given resource pool is over-utilized, the server can send a
CB_RECALL_ANY to clients holding recallable objects of the types
involved, allowing it to keep a certain number of such objects and
return any excess.
^^^^^^^^^^^^^^^^^
...
RCA4_TYPE_MASK_FILE_LAYOUT
The client is to return layouts of type LAYOUT4_NFSV4_1_FILES.
^^^^^^^^^^^^^^^^^
Isn't that explicit enough?
>
> In any case, there is no reason to make a difference between block,
> object and file layouts when it comes to CB_RECALL_ANY. The code to
> handle it should be the same for all.
>
Agreed.
Benny
Trond Myklebust wrote:
I don't necessarily disagree with what you are saying, but I have yet to
see a single server side implementation of CB_RECALL_ANY, let alone any
numbers that indicate performance or responsiveness problems resulting
from our existing client-side implementation.
I therefore find it hard to understand why optimising this particular
code is such a high priority, or why a patch that is adding per-file
layoutreturns to initiate_bulk_draining() is going to help anything at
all.
Testing between the linux block layout client and the EMC block layout
server revealed a deadlock when the server had handed out some number of
layouts and couldn't hand out any more. So now the EMC block layout server
implements CB_RECALL_ANY. So yes, this solves a real world problem, and
yes, there is a server that implements this.
We had some discussions at the time, and I don't remember if those were on
the linux-nfs list or in some other forum. We decided that the client was
in the best position to decide which layouts were no longer needed, so we
needed some way for the server to tell the client to return some layouts
without specifying which ones. CB_RECALL_ANY seemed custom-made for this
purpose, so we used it.
I don't think it would be appropriate for the server to recall all layouts
when it only needs some of them back.
On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
> For blocklayout, we need to issue layoutreturn to return layouts when
> handling CB_RECALL_ANY.
Why?
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
On Mon, 2011-10-31 at 19:08 +0200, Benny Halevy wrote:
> On 2011-10-31 18:45, Trond Myklebust wrote:
> > On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
> >> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
> >> <[email protected]> wrote:
> >>> On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
> >>>> For blocklayout, we need to issue layoutreturn to return layouts when
> >>>> handling CB_RECALL_ANY.
> >>>
> >>> Why?
> >> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
> >> that server wants client to return layout. And server will be waiting
> >> for layoutreturn in such case.
> >
> > No it doesn't. NFS4_OK means that the client acknowledges that it has
> > been given a new limit on the number of recallable objects it can keep.
> > There is no requirement in the text that it should send layoutreturn or
> > that the server should expect that.
>
> The motivation for CB_RECALL_ANY is to reduce the state on the *server* side.
> Quoting from RFC5661:
> The server may decide that it cannot hold all of the state for
> recallable objects, such as delegations and layouts, without running
> out of resources. In such a case, while not optimal, the server is
> free to recall individual objects to reduce the load.
> ...
> In order to implement an effective reclaim scheme for such objects,
> the server's knowledge of available resources must be used to
> determine when objects must be recalled with the clients selecting
> the actual objects to be returned.
> ^^^^^^^^^^^^^^
> ...
> When a given resource pool is over-utilized, the server can send a
> CB_RECALL_ANY to clients holding recallable objects of the types
> involved, allowing it to keep a certain number of such objects and
> return any excess.
> ^^^^^^^^^^^^^^^^^
> ...
> RCA4_TYPE_MASK_FILE_LAYOUT
>
> The client is to return layouts of type LAYOUT4_NFSV4_1_FILES.
> ^^^^^^^^^^^^^^^^^
>
> Isn't that explicit enough?
Leaving aside the fact that the above quotes contain no normative
language:
Right now, we do a bulk return of all layouts. Doing a layoutreturn for
each and every layout in that case is just ridiculous. Either do a
LAYOUTRETURN4_ALL after freeing all the layouts, or don't do anything at
all and just wait for the server to revoke the layouts for us (which is
what we currently do).
Both options should be faster than doing a LAYOUTRETURN4_FILE on each
and every file that is currently in use.
Trond
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
Hi,
Such a server implementation will certainly not be long in coming.
Matt
----- "Trond Myklebust" <[email protected]> wrote:
> On Mon, 2011-10-31 at 19:57 +0200, Benny Halevy wrote:
> >
> > Waiting for revocation may work well with some servers but would be
> disastrous in
> > terms of performance and responsiveness with others.
>
> I don't necessarily disagree with what you are saying, but I have yet
> to
> see a single server side implementation of CB_RECALL_ANY, let alone
> any
> numbers that indicate performance or responsiveness problems
> resulting
> from our existing client-side implementation.
>
> I therefore find it hard to understand why optimising this particular
> code is such a high priority, or why a patch that is adding per-file
> layoutreturns to initiate_bulk_draining() is going to help anything
> at
> all.
>
> Trond
>
--
Matt Benjamin
The Linux Box
206 South Fifth Ave. Suite 150
Ann Arbor, MI 48104
http://linuxbox.com
tel. 734-761-4689
fax. 734-769-8938
cel. 734-216-5309
On Mon, 2011-10-31 at 14:23 -0400, Matt W. Benjamin wrote:
> Hi,
>
> Such a server implementation will certainly not be long in coming.
Good. However until then, we have nothing to test any performance claims
against.
Trond
> ----- "Trond Myklebust" <[email protected]> wrote:
>
> > On Mon, 2011-10-31 at 19:57 +0200, Benny Halevy wrote:
>
> > >
> > > Waiting for revocation may work well with some servers but would be
> > disastrous in
> > > terms of performance and responsiveness with others.
> >
> > I don't necessarily disagree with what you are saying, but I have yet
> > to
> > see a single server side implementation of CB_RECALL_ANY, let alone
> > any
> > numbers that indicate performance or responsiveness problems
> > resulting
> > from our existing client-side implementation.
> >
> > I therefore find it hard to understand why optimising this particular
> > code is such a high priority, or why a patch that is adding per-file
> > layoutreturns to initiate_bulk_draining() is going to help anything
> > at
> > all.
> >
> > Trond
> >
>
>
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
On 2011-10-31 19:42, Trond Myklebust wrote:
> On Mon, 2011-10-31 at 19:08 +0200, Benny Halevy wrote:
>> On 2011-10-31 18:45, Trond Myklebust wrote:
>>> On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
>>>> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
>>>> <[email protected]> wrote:
>>>>> On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
>>>>>> For blocklayout, we need to issue layoutreturn to return layouts when
>>>>>> handling CB_RECALL_ANY.
>>>>>
>>>>> Why?
>>>> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
>>>> that server wants client to return layout. And server will be waiting
>>>> for layoutreturn in such case.
>>>
>>> No it doesn't. NFS4_OK means that the client acknowledges that it has
>>> been given a new limit on the number of recallable objects it can keep.
>>> There is no requirement in the text that it should send layoutreturn or
>>> that the server should expect that.
>>
>> The motivation for CB_RECALL_ANY is to reduce the state on the *server* side.
>> Quoting from RFC5661:
>> The server may decide that it cannot hold all of the state for
>> recallable objects, such as delegations and layouts, without running
>> out of resources. In such a case, while not optimal, the server is
>> free to recall individual objects to reduce the load.
>> ...
>> In order to implement an effective reclaim scheme for such objects,
>> the server's knowledge of available resources must be used to
>> determine when objects must be recalled with the clients selecting
>> the actual objects to be returned.
>> ^^^^^^^^^^^^^^
>> ...
>> When a given resource pool is over-utilized, the server can send a
>> CB_RECALL_ANY to clients holding recallable objects of the types
>> involved, allowing it to keep a certain number of such objects and
>> return any excess.
>> ^^^^^^^^^^^^^^^^^
>> ...
>> RCA4_TYPE_MASK_FILE_LAYOUT
>>
>> The client is to return layouts of type LAYOUT4_NFSV4_1_FILES.
>> ^^^^^^^^^^^^^^^^^
>>
>> Isn't that explicit enough?
>
> Leaving aside the fact that the above quotes contain no normative
> language:
> Right now, we do a bulk return of all layouts. Doing a layoutreturn for
> each and every layout in that case is just ridiculous. Either do a
The idea is to return the layouts for files that are the least used,
not each and every layout.
> LAYOUTRETURN4_ALL after freeing all the layouts, or don't do anything at
> all and just wait for the server to revoke the layouts for us (which is
> what we currently do).
> Both options should be faster than doing a LAYOUTRETURN4_FILE on each
> and every file that is currently in use.
Doing LAYOUTRETURN4_ALL might cause a bug hiccup if the client needs to then send
a LAYOUTGET for each and every file that *is* currently in use.
So serving a CB_RECALL_ANY keeping more than 50% of the recallable objects means
the client would be better off returning the excess rather than returning everything
and reclaiming > 50% back.
Waiting for revocation may work well with some servers but would be disastrous in
terms of performance and responsiveness with others.
Benny
>
> Trond
>
On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
<[email protected]> wrote:
> On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
>> For blocklayout, we need to issue layoutreturn to return layouts when
>> handling CB_RECALL_ANY.
>
> Why?
Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
that server wants client to return layout. And server will be waiting
for layoutreturn in such case.
Thanks,
Tao
>
> --
> Trond Myklebust
> Linux NFS client maintainer
>
> NetApp
> [email protected]
> http://www.netapp.com
>
>
On Mon, 2011-10-31 at 14:31 -0400, Jim Rees wrote:
> Trond Myklebust wrote:
>
> I don't necessarily disagree with what you are saying, but I have yet to
> see a single server side implementation of CB_RECALL_ANY, let alone any
> numbers that indicate performance or responsiveness problems resulting
> from our existing client-side implementation.
>
> I therefore find it hard to understand why optimising this particular
> code is such a high priority, or why a patch that is adding per-file
> layoutreturns to initiate_bulk_draining() is going to help anything at
> all.
>
> Testing between the linux block layout client and the EMC block layout
> server revealed a deadlock when the server had handed out some number of
> layouts and couldn't hand out any more. So now the EMC block layout server
> implements CB_RECALL_ANY. So yes, this solves a real world problem, and
> yes, there is a server that implements this.
>
> We had some discussions at the time, and I don't remember if those were on
> the linux-nfs list or in some other forum. We decided that the client was
> in the best position to decide which layouts were no longer needed, so we
> needed some way for the server to tell the client to return some layouts
> without specifying which ones. CB_RECALL_ANY seemed custom-made for this
> purpose, so we used it.
>
> I don't think it would be appropriate for the server to recall all layouts
> when it only needs some of them back.
As I said previously: the current client implementation deals with
CB_RECALL_ANY by calling initiate_file_draining(), which forgets _all_
layouts. If that is what we want to continue to use, then sending
layoutreturn with LAYOUTRETURN4_ALL is appropriate.
Otherwise, please fix the client to be more selective (in which case
LAYOUTRETURN4_FILE may be more appropriate) and please remember to
provide performance numbers to justify the need for optimisation.
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
On Mon, 2011-10-31 at 19:57 +0200, Benny Halevy wrote:
> On 2011-10-31 19:42, Trond Myklebust wrote:
> > On Mon, 2011-10-31 at 19:08 +0200, Benny Halevy wrote:
> >> On 2011-10-31 18:45, Trond Myklebust wrote:
> >>> On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
> >>>> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
> >>>> <[email protected]> wrote:
> >>>>> On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
> >>>>>> For blocklayout, we need to issue layoutreturn to return layouts when
> >>>>>> handling CB_RECALL_ANY.
> >>>>>
> >>>>> Why?
> >>>> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
> >>>> that server wants client to return layout. And server will be waiting
> >>>> for layoutreturn in such case.
> >>>
> >>> No it doesn't. NFS4_OK means that the client acknowledges that it has
> >>> been given a new limit on the number of recallable objects it can keep.
> >>> There is no requirement in the text that it should send layoutreturn or
> >>> that the server should expect that.
> >>
> >> The motivation for CB_RECALL_ANY is to reduce the state on the *server* side.
> >> Quoting from RFC5661:
> >> The server may decide that it cannot hold all of the state for
> >> recallable objects, such as delegations and layouts, without running
> >> out of resources. In such a case, while not optimal, the server is
> >> free to recall individual objects to reduce the load.
> >> ...
> >> In order to implement an effective reclaim scheme for such objects,
> >> the server's knowledge of available resources must be used to
> >> determine when objects must be recalled with the clients selecting
> >> the actual objects to be returned.
> >> ^^^^^^^^^^^^^^
> >> ...
> >> When a given resource pool is over-utilized, the server can send a
> >> CB_RECALL_ANY to clients holding recallable objects of the types
> >> involved, allowing it to keep a certain number of such objects and
> >> return any excess.
> >> ^^^^^^^^^^^^^^^^^
> >> ...
> >> RCA4_TYPE_MASK_FILE_LAYOUT
> >>
> >> The client is to return layouts of type LAYOUT4_NFSV4_1_FILES.
> >> ^^^^^^^^^^^^^^^^^
> >>
> >> Isn't that explicit enough?
> >
> > Leaving aside the fact that the above quotes contain no normative
> > language:
> > Right now, we do a bulk return of all layouts. Doing a layoutreturn for
> > each and every layout in that case is just ridiculous. Either do a
>
> The idea is to return the layouts for files that are the least used,
> not each and every layout.
>
> > LAYOUTRETURN4_ALL after freeing all the layouts, or don't do anything at
> > all and just wait for the server to revoke the layouts for us (which is
> > what we currently do).
> > Both options should be faster than doing a LAYOUTRETURN4_FILE on each
> > and every file that is currently in use.
>
> Doing LAYOUTRETURN4_ALL might cause a bug hiccup if the client needs to then send
> a LAYOUTGET for each and every file that *is* currently in use.
> So serving a CB_RECALL_ANY keeping more than 50% of the recallable objects means
> the client would be better off returning the excess rather than returning everything
> and reclaiming > 50% back.
>
> Waiting for revocation may work well with some servers but would be disastrous in
> terms of performance and responsiveness with others.
I don't necessarily disagree with what you are saying, but I have yet to
see a single server side implementation of CB_RECALL_ANY, let alone any
numbers that indicate performance or responsiveness problems resulting
from our existing client-side implementation.
I therefore find it hard to understand why optimising this particular
code is such a high priority, or why a patch that is adding per-file
layoutreturns to initiate_bulk_draining() is going to help anything at
all.
Trond
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
On Tue, 2011-11-01 at 00:38 +0800, Peng Tao wrote:
> On Mon, Oct 31, 2011 at 11:49 PM, Trond Myklebust
> <[email protected]> wrote:
> > On Mon, 2011-10-31 at 08:15 -0700, Peng Tao wrote:
> >> For blocklayout, we need to issue layoutreturn to return layouts when
> >> handling CB_RECALL_ANY.
> >
> > Why?
> Because replying NFS4_OK to CB_RECALL_ANY indicates that client knows
> that server wants client to return layout. And server will be waiting
> for layoutreturn in such case.
No it doesn't. NFS4_OK means that the client acknowledges that it has
been given a new limit on the number of recallable objects it can keep.
There is no requirement in the text that it should send layoutreturn or
that the server should expect that.
In any case, there is no reason to make a difference between block,
object and file layouts when it comes to CB_RECALL_ANY. The code to
handle it should be the same for all.
--
Trond Myklebust
Linux NFS client maintainer
NetApp
[email protected]
http://www.netapp.com
As I recall, the primary focus of CB_RECALL ANY was in dealing with recall of delegations.
With delegations the server is in no position to, if needs to get some number back, pick
ones that are not being used, since it is in the nature of delegations that the client may
be using them with great benefit with the server not hearing anything about them. So the
logic there was that if the client didn't do the picking and just did nothing the server would
have to pick for himself, and would almost certainly do a poor job, and the client might not
like the server's choices.
For other recallable objects, the logic my be different. In the case of layouts, the
data servers will know if they are being used but the MDFS will not. So if the client won't
act on this, and the server picks badly, it can't say "there was no way I could have picked
better ones" as it can with delegations.
The spec should make it clear whether the client is at fault in this case or the server
should have done its best by incorporating data server information on layout use, but ....
The chances of client and server implementors agreeing on this are fairly low.
So ambiguity/confusion on this issue may persist for a while.
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Welch, Brent
Sent: Monday, October 31, 2011 5:43 PM
To: Jim Rees; Trond Myklebust
Cc: [email protected]; nfsv4 list
Subject: Re: [nfsv4] [PATCH 2/2] nfs41: handle BLK_LAYOUT CB_RECALL_ANY
CB_RECALL_ANY was added based on Panasas experience with its own protocols. When you have
100's (or 1000's) of clients and millions of files for which you maintain state, it is
very much worth optimizing how the server can reclaim this state.
Brent
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Jim Rees
Sent: Monday, October 31, 2011 11:32 AM
To: Trond Myklebust
Cc: [email protected]; nfsv4 list
Subject: Re: [nfsv4] [PATCH 2/2] nfs41: handle BLK_LAYOUT CB_RECALL_ANY
Trond Myklebust wrote:
I don't necessarily disagree with what you are saying, but I have yet to
see a single server side implementation of CB_RECALL_ANY, let alone any
numbers that indicate performance or responsiveness problems resulting
from our existing client-side implementation.
I therefore find it hard to understand why optimising this particular
code is such a high priority, or why a patch that is adding per-file
layoutreturns to initiate_bulk_draining() is going to help anything at
all.
Testing between the linux block layout client and the EMC block layout
server revealed a deadlock when the server had handed out some number of
layouts and couldn't hand out any more. So now the EMC block layout server
implements CB_RECALL_ANY. So yes, this solves a real world problem, and
yes, there is a server that implements this.
We had some discussions at the time, and I don't remember if those were on
the linux-nfs list or in some other forum. We decided that the client was
in the best position to decide which layouts were no longer needed, so we
needed some way for the server to tell the client to return some layouts
without specifying which ones. CB_RECALL_ANY seemed custom-made for this
purpose, so we used it.
I don't think it would be appropriate for the server to recall all layouts
when it only needs some of them back.
_______________________________________________
nfsv4 mailing list
[email protected]
https://www.ietf.org/mailman/listinfo/nfsv4
_______________________________________________
nfsv4 mailing list
[email protected]
https://www.ietf.org/mailman/listinfo/nfsv4
SGksIA0KDQo+IC0tLS0tT3JpZ2luYWwgTWVzc2FnZS0tLS0tDQo+IEZyb206IGxpbnV4LW5mcy1v
d25lckB2Z2VyLmtlcm5lbC5vcmcgW21haWx0bzpsaW51eC1uZnMtb3duZXJAdmdlci5rZXJuZWwu
b3JnXQ0KPiBPbiBCZWhhbGYgT2YgdGFvLnBlbmdAZW1jLmNvbQ0KPiBTZW50OiBUdWVzZGF5LCBO
b3ZlbWJlciAwMSwgMjAxMSAxOjI5IFBNDQo+IFRvOiBUcm9uZC5NeWtsZWJ1c3RAbmV0YXBwLmNv
bTsgcmVlc0B1bWljaC5lZHUNCj4gQ2M6IGJoYWxldnlAdG9uaWFuLmNvbTsgYmVyZ3dvbGZAZ21h
aWwuY29tOyBsaW51eC1uZnNAdmdlci5rZXJuZWwub3JnOw0KPiBuZnN2NEBpZXRmLm9yZw0KPiBT
dWJqZWN0OiBSRTogW1BBVENIIDIvMl0gbmZzNDE6IGhhbmRsZSBCTEtfTEFZT1VUIENCX1JFQ0FM
TF9BTlkNCj4gDQo+IEhpLCBUcm9uZCwNCj4gDQo+ID4gLS0tLS1PcmlnaW5hbCBNZXNzYWdlLS0t
LS0NCj4gPiBGcm9tOiBUcm9uZCBNeWtsZWJ1c3QgW21haWx0bzpUcm9uZC5NeWtsZWJ1c3RAbmV0
YXBwLmNvbV0NCj4gPiBTZW50OiBUdWVzZGF5LCBOb3ZlbWJlciAwMSwgMjAxMSAyOjQwIEFNDQo+
ID4gVG86IEppbSBSZWVzDQo+ID4gQ2M6IEJlbm55IEhhbGV2eTsgUGVuZyBUYW87IGxpbnV4LW5m
c0B2Z2VyLmtlcm5lbC5vcmc7IFBlbmcsIFRhbzsgbmZzdjQgbGlzdA0KPiA+IFN1YmplY3Q6IFJl
OiBbUEFUQ0ggMi8yXSBuZnM0MTogaGFuZGxlIEJMS19MQVlPVVQgQ0JfUkVDQUxMX0FOWQ0KPiA+
DQo+ID4gT24gTW9uLCAyMDExLTEwLTMxIGF0IDE0OjMxIC0wNDAwLCBKaW0gUmVlcyB3cm90ZToN
Cj4gPiA+IFRyb25kIE15a2xlYnVzdCB3cm90ZToNCj4gPiA+DQo+ID4gPiAgIEkgZG9uJ3QgbmVj
ZXNzYXJpbHkgZGlzYWdyZWUgd2l0aCB3aGF0IHlvdSBhcmUgc2F5aW5nLCBidXQgSSBoYXZlIHll
dCB0bw0KPiA+ID4gICBzZWUgYSBzaW5nbGUgc2VydmVyIHNpZGUgaW1wbGVtZW50YXRpb24gb2Yg
Q0JfUkVDQUxMX0FOWSwgbGV0IGFsb25lIGFueQ0KPiA+ID4gICBudW1iZXJzIHRoYXQgaW5kaWNh
dGUgcGVyZm9ybWFuY2Ugb3IgcmVzcG9uc2l2ZW5lc3MgcHJvYmxlbXMgcmVzdWx0aW5nDQo+ID4g
PiAgIGZyb20gb3VyIGV4aXN0aW5nIGNsaWVudC1zaWRlIGltcGxlbWVudGF0aW9uLg0KPiA+ID4N
Cj4gPiA+ICAgSSB0aGVyZWZvcmUgZmluZCBpdCBoYXJkIHRvIHVuZGVyc3RhbmQgd2h5IG9wdGlt
aXNpbmcgdGhpcyBwYXJ0aWN1bGFyDQo+ID4gPiAgIGNvZGUgaXMgc3VjaCBhIGhpZ2ggcHJpb3Jp
dHksIG9yIHdoeSBhIHBhdGNoIHRoYXQgaXMgYWRkaW5nIHBlci1maWxlDQo+ID4gPiAgIGxheW91
dHJldHVybnMgdG8gaW5pdGlhdGVfYnVsa19kcmFpbmluZygpIGlzIGdvaW5nIHRvIGhlbHAgYW55
dGhpbmcgYXQNCj4gPiA+ICAgYWxsLg0KPiA+ID4NCj4gPiA+IFRlc3RpbmcgYmV0d2VlbiB0aGUg
bGludXggYmxvY2sgbGF5b3V0IGNsaWVudCBhbmQgdGhlIEVNQyBibG9jayBsYXlvdXQNCj4gPiA+
IHNlcnZlciByZXZlYWxlZCBhIGRlYWRsb2NrIHdoZW4gdGhlIHNlcnZlciBoYWQgaGFuZGVkIG91
dCBzb21lIG51bWJlciBvZg0KPiA+ID4gbGF5b3V0cyBhbmQgY291bGRuJ3QgaGFuZCBvdXQgYW55
IG1vcmUuICBTbyBub3cgdGhlIEVNQyBibG9jayBsYXlvdXQgc2VydmVyDQo+ID4gPiBpbXBsZW1l
bnRzIENCX1JFQ0FMTF9BTlkuICBTbyB5ZXMsIHRoaXMgc29sdmVzIGEgcmVhbCB3b3JsZCBwcm9i
bGVtLCBhbmQNCj4gPiA+IHllcywgdGhlcmUgaXMgYSBzZXJ2ZXIgdGhhdCBpbXBsZW1lbnRzIHRo
aXMuDQo+ID4gPg0KPiA+ID4gV2UgaGFkIHNvbWUgZGlzY3Vzc2lvbnMgYXQgdGhlIHRpbWUsIGFu
ZCBJIGRvbid0IHJlbWVtYmVyIGlmIHRob3NlIHdlcmUgb24NCj4gPiA+IHRoZSBsaW51eC1uZnMg
bGlzdCBvciBpbiBzb21lIG90aGVyIGZvcnVtLiAgV2UgZGVjaWRlZCB0aGF0IHRoZSBjbGllbnQg
d2FzDQo+ID4gPiBpbiB0aGUgYmVzdCBwb3NpdGlvbiB0byBkZWNpZGUgd2hpY2ggbGF5b3V0cyB3
ZXJlIG5vIGxvbmdlciBuZWVkZWQsIHNvIHdlDQo+ID4gPiBuZWVkZWQgc29tZSB3YXkgZm9yIHRo
ZSBzZXJ2ZXIgdG8gdGVsbCB0aGUgY2xpZW50IHRvIHJldHVybiBzb21lIGxheW91dHMNCj4gPiA+
IHdpdGhvdXQgc3BlY2lmeWluZyB3aGljaCBvbmVzLiAgQ0JfUkVDQUxMX0FOWSBzZWVtZWQgY3Vz
dG9tLW1hZGUgZm9yIHRoaXMNCj4gPiA+IHB1cnBvc2UsIHNvIHdlIHVzZWQgaXQuDQo+ID4gPg0K
PiA+ID4gSSBkb24ndCB0aGluayBpdCB3b3VsZCBiZSBhcHByb3ByaWF0ZSBmb3IgdGhlIHNlcnZl
ciB0byByZWNhbGwgYWxsIGxheW91dHMNCj4gPiA+IHdoZW4gaXQgb25seSBuZWVkcyBzb21lIG9m
IHRoZW0gYmFjay4NCj4gPg0KPiA+IEFzIEkgc2FpZCBwcmV2aW91c2x5OiB0aGUgY3VycmVudCBj
bGllbnQgaW1wbGVtZW50YXRpb24gZGVhbHMgd2l0aA0KPiA+IENCX1JFQ0FMTF9BTlkgYnkgY2Fs
bGluZyBpbml0aWF0ZV9maWxlX2RyYWluaW5nKCksIHdoaWNoIGZvcmdldHMgX2FsbF8NCj4gPiBs
YXlvdXRzLiBJZiB0aGF0IGlzIHdoYXQgd2Ugd2FudCB0byBjb250aW51ZSB0byB1c2UsIHRoZW4g
c2VuZGluZw0KPiA+IGxheW91dHJldHVybiB3aXRoIExBWU9VVFJFVFVSTjRfQUxMIGlzIGFwcHJv
cHJpYXRlLg0KPiA+IE90aGVyd2lzZSwgcGxlYXNlIGZpeCB0aGUgY2xpZW50IHRvIGJlIG1vcmUg
c2VsZWN0aXZlIChpbiB3aGljaCBjYXNlDQo+ID4gTEFZT1VUUkVUVVJONF9GSUxFIG1heSBiZSBt
b3JlIGFwcHJvcHJpYXRlKSBhbmQgcGxlYXNlIHJlbWVtYmVyIHRvDQo+ID4gcHJvdmlkZSBwZXJm
b3JtYW5jZSBudW1iZXJzIHRvIGp1c3RpZnkgdGhlIG5lZWQgZm9yIG9wdGltaXNhdGlvbi4NCj4g
T0ssIGZhaXIgZW5vdWdoLiBJIHdpbGwgd29yayBvbiB0aGUgc2VsZWN0aXZlIGxheW91dHJldHVy
biBhcHByb2FjaCB3aGVuIEkgaGF2ZSB0aGUNCj4gdGltZSBzbG90LiBBbmQgSSBhZ3JlZSBpdCBp
cyBvZiBsb3cgcHJpb3JpdHkuDQpJIGhlYXJkIHRoYXQgdGhlcmUgYXJlIHNvbWUgZGlzY3Vzc2lv
bnMgYWJvdXQgdGhpcyBpbiBsYXN0IHBuZnMgTGludXggc3RhdHVzIG1lZXRpbmcuIFNvcnJ5IEkg
d2Fzbid0IGluIHRoZSBtZWV0aW5nIGJ1dCBwbGVhc2Ugc29tZW9uZSB1cGRhdGUgYnJpZWZseSB3
aGF0IHRoZSBjb25jbHVzaW9uIGlzPyBJcyB0aGUgYWJvdmUgc29sdXRpb24gc3RpbGwgdmFsaWQ/
IEp1c3QgZG9uJ3Qgd2FudCB0byBnbyB3cm9uZyBkaXJlY3Rpb24gYWdhaW4uLi4gOi1QDQoNClRo
YW5rcywNClRhbw0K
SGksIFRyb25kLA0KDQo+IC0tLS0tT3JpZ2luYWwgTWVzc2FnZS0tLS0tDQo+IEZyb206IFRyb25k
IE15a2xlYnVzdCBbbWFpbHRvOlRyb25kLk15a2xlYnVzdEBuZXRhcHAuY29tXQ0KPiBTZW50OiBU
dWVzZGF5LCBOb3ZlbWJlciAwMSwgMjAxMSAyOjQwIEFNDQo+IFRvOiBKaW0gUmVlcw0KPiBDYzog
QmVubnkgSGFsZXZ5OyBQZW5nIFRhbzsgbGludXgtbmZzQHZnZXIua2VybmVsLm9yZzsgUGVuZywg
VGFvOyBuZnN2NCBsaXN0DQo+IFN1YmplY3Q6IFJlOiBbUEFUQ0ggMi8yXSBuZnM0MTogaGFuZGxl
IEJMS19MQVlPVVQgQ0JfUkVDQUxMX0FOWQ0KPiANCj4gT24gTW9uLCAyMDExLTEwLTMxIGF0IDE0
OjMxIC0wNDAwLCBKaW0gUmVlcyB3cm90ZToNCj4gPiBUcm9uZCBNeWtsZWJ1c3Qgd3JvdGU6DQo+
ID4NCj4gPiAgIEkgZG9uJ3QgbmVjZXNzYXJpbHkgZGlzYWdyZWUgd2l0aCB3aGF0IHlvdSBhcmUg
c2F5aW5nLCBidXQgSSBoYXZlIHlldCB0bw0KPiA+ICAgc2VlIGEgc2luZ2xlIHNlcnZlciBzaWRl
IGltcGxlbWVudGF0aW9uIG9mIENCX1JFQ0FMTF9BTlksIGxldCBhbG9uZSBhbnkNCj4gPiAgIG51
bWJlcnMgdGhhdCBpbmRpY2F0ZSBwZXJmb3JtYW5jZSBvciByZXNwb25zaXZlbmVzcyBwcm9ibGVt
cyByZXN1bHRpbmcNCj4gPiAgIGZyb20gb3VyIGV4aXN0aW5nIGNsaWVudC1zaWRlIGltcGxlbWVu
dGF0aW9uLg0KPiA+DQo+ID4gICBJIHRoZXJlZm9yZSBmaW5kIGl0IGhhcmQgdG8gdW5kZXJzdGFu
ZCB3aHkgb3B0aW1pc2luZyB0aGlzIHBhcnRpY3VsYXINCj4gPiAgIGNvZGUgaXMgc3VjaCBhIGhp
Z2ggcHJpb3JpdHksIG9yIHdoeSBhIHBhdGNoIHRoYXQgaXMgYWRkaW5nIHBlci1maWxlDQo+ID4g
ICBsYXlvdXRyZXR1cm5zIHRvIGluaXRpYXRlX2J1bGtfZHJhaW5pbmcoKSBpcyBnb2luZyB0byBo
ZWxwIGFueXRoaW5nIGF0DQo+ID4gICBhbGwuDQo+ID4NCj4gPiBUZXN0aW5nIGJldHdlZW4gdGhl
IGxpbnV4IGJsb2NrIGxheW91dCBjbGllbnQgYW5kIHRoZSBFTUMgYmxvY2sgbGF5b3V0DQo+ID4g
c2VydmVyIHJldmVhbGVkIGEgZGVhZGxvY2sgd2hlbiB0aGUgc2VydmVyIGhhZCBoYW5kZWQgb3V0
IHNvbWUgbnVtYmVyIG9mDQo+ID4gbGF5b3V0cyBhbmQgY291bGRuJ3QgaGFuZCBvdXQgYW55IG1v
cmUuICBTbyBub3cgdGhlIEVNQyBibG9jayBsYXlvdXQgc2VydmVyDQo+ID4gaW1wbGVtZW50cyBD
Ql9SRUNBTExfQU5ZLiAgU28geWVzLCB0aGlzIHNvbHZlcyBhIHJlYWwgd29ybGQgcHJvYmxlbSwg
YW5kDQo+ID4geWVzLCB0aGVyZSBpcyBhIHNlcnZlciB0aGF0IGltcGxlbWVudHMgdGhpcy4NCj4g
Pg0KPiA+IFdlIGhhZCBzb21lIGRpc2N1c3Npb25zIGF0IHRoZSB0aW1lLCBhbmQgSSBkb24ndCBy
ZW1lbWJlciBpZiB0aG9zZSB3ZXJlIG9uDQo+ID4gdGhlIGxpbnV4LW5mcyBsaXN0IG9yIGluIHNv
bWUgb3RoZXIgZm9ydW0uICBXZSBkZWNpZGVkIHRoYXQgdGhlIGNsaWVudCB3YXMNCj4gPiBpbiB0
aGUgYmVzdCBwb3NpdGlvbiB0byBkZWNpZGUgd2hpY2ggbGF5b3V0cyB3ZXJlIG5vIGxvbmdlciBu
ZWVkZWQsIHNvIHdlDQo+ID4gbmVlZGVkIHNvbWUgd2F5IGZvciB0aGUgc2VydmVyIHRvIHRlbGwg
dGhlIGNsaWVudCB0byByZXR1cm4gc29tZSBsYXlvdXRzDQo+ID4gd2l0aG91dCBzcGVjaWZ5aW5n
IHdoaWNoIG9uZXMuICBDQl9SRUNBTExfQU5ZIHNlZW1lZCBjdXN0b20tbWFkZSBmb3IgdGhpcw0K
PiA+IHB1cnBvc2UsIHNvIHdlIHVzZWQgaXQuDQo+ID4NCj4gPiBJIGRvbid0IHRoaW5rIGl0IHdv
dWxkIGJlIGFwcHJvcHJpYXRlIGZvciB0aGUgc2VydmVyIHRvIHJlY2FsbCBhbGwgbGF5b3V0cw0K
PiA+IHdoZW4gaXQgb25seSBuZWVkcyBzb21lIG9mIHRoZW0gYmFjay4NCj4gDQo+IEFzIEkgc2Fp
ZCBwcmV2aW91c2x5OiB0aGUgY3VycmVudCBjbGllbnQgaW1wbGVtZW50YXRpb24gZGVhbHMgd2l0
aA0KPiBDQl9SRUNBTExfQU5ZIGJ5IGNhbGxpbmcgaW5pdGlhdGVfZmlsZV9kcmFpbmluZygpLCB3
aGljaCBmb3JnZXRzIF9hbGxfDQo+IGxheW91dHMuIElmIHRoYXQgaXMgd2hhdCB3ZSB3YW50IHRv
IGNvbnRpbnVlIHRvIHVzZSwgdGhlbiBzZW5kaW5nDQo+IGxheW91dHJldHVybiB3aXRoIExBWU9V
VFJFVFVSTjRfQUxMIGlzIGFwcHJvcHJpYXRlLg0KPiBPdGhlcndpc2UsIHBsZWFzZSBmaXggdGhl
IGNsaWVudCB0byBiZSBtb3JlIHNlbGVjdGl2ZSAoaW4gd2hpY2ggY2FzZQ0KPiBMQVlPVVRSRVRV
Uk40X0ZJTEUgbWF5IGJlIG1vcmUgYXBwcm9wcmlhdGUpIGFuZCBwbGVhc2UgcmVtZW1iZXIgdG8N
Cj4gcHJvdmlkZSBwZXJmb3JtYW5jZSBudW1iZXJzIHRvIGp1c3RpZnkgdGhlIG5lZWQgZm9yIG9w
dGltaXNhdGlvbi4NCk9LLCBmYWlyIGVub3VnaC4gSSB3aWxsIHdvcmsgb24gdGhlIHNlbGVjdGl2
ZSBsYXlvdXRyZXR1cm4gYXBwcm9hY2ggd2hlbiBJIGhhdmUgdGhlIHRpbWUgc2xvdC4gQW5kIEkg
YWdyZWUgaXQgaXMgb2YgbG93IHByaW9yaXR5Lg0KDQpUaGFua3MsDQpUYW8NCg0K