From: Anna Schumaker <[email protected]>
These patches are a handul of fixes I've done recently to the READ_PLUS
code. This includes fixing some smatch warnings, fixing the XDR reply
length calculation, improving scratch buffer handling, and having
xdr_inline_decode() kmap pages if we detect that they're HIGHMEM so we
don't oops during READ_PLUS xdr decoding.
I also (optimistically) have a patch at the end to enable
CONFIG_READ_PLUS by default. My hope right now is that we can enable
READ_PLUS for Linux 6.6, and remove it entirely in 6.7 if all goes well.
Thoughts? Patch #4 probably needs the most review, and I'm open to other
approaches if something else makes more sense!
Thanks,
Anna
Anna Schumaker (5):
NFSv4.2: Fix READ_PLUS smatch warnings
NFSv4.2: Fix READ_PLUS size calculations
NFSv4.2: Rework scratch handling for READ_PLUS (again)
SUNRPC: kmap() the xdr pages during decode
NFS: Enable the READ_PLUS operation by default
fs/nfs/Kconfig | 6 ++----
fs/nfs/internal.h | 1 +
fs/nfs/nfs42.h | 1 +
fs/nfs/nfs42xdr.c | 17 +++++++++++------
fs/nfs/nfs4proc.c | 13 +------------
fs/nfs/read.c | 10 ++++++++++
include/linux/sunrpc/xdr.h | 3 +++
net/sunrpc/clnt.c | 1 +
net/sunrpc/svc.c | 2 ++
net/sunrpc/xdr.c | 27 ++++++++++++++++++++++++++-
10 files changed, 58 insertions(+), 23 deletions(-)
--
2.41.0
From: Anna Schumaker <[email protected]>
I bump the decode_read_plus_maxsz to account for hole segments, but I
need to subtract out this increase when calling
rpc_prepare_reply_pages() so the common case of single data segment
replies can be directly placed into the xdr pages without needing to be
shifted around.
Reported-by: Chuck Lever <[email protected]>
Fixes: d3b00a802c845 ("NFS: Replace the READ_PLUS decoding code")
Signed-off-by: Anna Schumaker <[email protected]>
---
fs/nfs/nfs42xdr.c | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c
index d0919c5bf61c..78193f04d892 100644
--- a/fs/nfs/nfs42xdr.c
+++ b/fs/nfs/nfs42xdr.c
@@ -54,10 +54,16 @@
(1 /* data_content4 */ + \
2 /* data_info4.di_offset */ + \
1 /* data_info4.di_length */)
+#define NFS42_READ_PLUS_HOLE_SEGMENT_SIZE \
+ (1 /* data_content4 */ + \
+ 2 /* data_info4.di_offset */ + \
+ 2 /* data_info4.di_length */)
+#define READ_PLUS_SEGMENT_SIZE_DIFF (NFS42_READ_PLUS_HOLE_SEGMENT_SIZE - \
+ NFS42_READ_PLUS_DATA_SEGMENT_SIZE)
#define decode_read_plus_maxsz (op_decode_hdr_maxsz + \
1 /* rpr_eof */ + \
1 /* rpr_contents count */ + \
- NFS42_READ_PLUS_DATA_SEGMENT_SIZE)
+ NFS42_READ_PLUS_HOLE_SEGMENT_SIZE)
#define encode_seek_maxsz (op_encode_hdr_maxsz + \
encode_stateid_maxsz + \
2 /* offset */ + \
@@ -617,8 +623,8 @@ static void nfs4_xdr_enc_read_plus(struct rpc_rqst *req,
encode_putfh(xdr, args->fh, &hdr);
encode_read_plus(xdr, args, &hdr);
- rpc_prepare_reply_pages(req, args->pages, args->pgbase,
- args->count, hdr.replen);
+ rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->count,
+ hdr.replen - READ_PLUS_SEGMENT_SIZE_DIFF);
encode_nops(&hdr);
}
--
2.41.0
From: Anna Schumaker <[email protected]>
I found that the read code might send multiple requests using the same
nfs_pgio_header, but nfs4_proc_read_setup() is only called once. This is
how we ended up occasionally double-freeing the scratch buffer, but also
means we set a NULL pointer but non-zero length to the xdr scratch
buffer. This results in an oops the first time decoding needs to copy
something to scratch, which frequently happens when decoding READ_PLUS
hole segments.
I fix this by moving scratch handling into the pageio read code. I
provide a function to allocate scratch space for decoding read replies,
and free the scratch buffer when the nfs_pgio_header is freed.
Fixes: fbd2a05f29a9 (NFSv4.2: Rework scratch handling for READ_PLUS)
Signed-off-by: Anna Schumaker <[email protected]>
---
fs/nfs/internal.h | 1 +
fs/nfs/nfs42.h | 1 +
fs/nfs/nfs42xdr.c | 2 +-
fs/nfs/nfs4proc.c | 13 +------------
fs/nfs/read.c | 10 ++++++++++
5 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 913c09806c7f..41abea340ad8 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -493,6 +493,7 @@ extern const struct nfs_pgio_completion_ops nfs_async_read_completion_ops;
extern void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
struct inode *inode, bool force_mds,
const struct nfs_pgio_completion_ops *compl_ops);
+extern bool nfs_read_alloc_scratch(struct nfs_pgio_header *hdr, size_t size);
extern int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio,
struct nfs_open_context *ctx,
struct folio *folio);
diff --git a/fs/nfs/nfs42.h b/fs/nfs/nfs42.h
index 0fe5aacbcfdf..b59876b01a1e 100644
--- a/fs/nfs/nfs42.h
+++ b/fs/nfs/nfs42.h
@@ -13,6 +13,7 @@
* more? Need to consider not to pre-alloc too much for a compound.
*/
#define PNFS_LAYOUTSTATS_MAXDEV (4)
+#define READ_PLUS_SCRATCH_SIZE (16)
/* nfs4.2proc.c */
#ifdef CONFIG_NFS_V4_2
diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c
index 78193f04d892..9e3ae53e2205 100644
--- a/fs/nfs/nfs42xdr.c
+++ b/fs/nfs/nfs42xdr.c
@@ -1433,7 +1433,7 @@ static int nfs4_xdr_dec_read_plus(struct rpc_rqst *rqstp,
struct compound_hdr hdr;
int status;
- xdr_set_scratch_buffer(xdr, res->scratch, sizeof(res->scratch));
+ xdr_set_scratch_buffer(xdr, res->scratch, READ_PLUS_SCRATCH_SIZE);
status = decode_compound_hdr(xdr, &hdr);
if (status)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index e1a886b58354..c2bdbcef5c6c 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -5438,18 +5438,8 @@ static bool nfs4_read_plus_not_supported(struct rpc_task *task,
return false;
}
-static inline void nfs4_read_plus_scratch_free(struct nfs_pgio_header *hdr)
-{
- if (hdr->res.scratch) {
- kfree(hdr->res.scratch);
- hdr->res.scratch = NULL;
- }
-}
-
static int nfs4_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
{
- nfs4_read_plus_scratch_free(hdr);
-
if (!nfs4_sequence_done(task, &hdr->res.seq_res))
return -EAGAIN;
if (nfs4_read_stateid_changed(task, &hdr->args))
@@ -5469,8 +5459,7 @@ static bool nfs42_read_plus_support(struct nfs_pgio_header *hdr,
/* Note: We don't use READ_PLUS with pNFS yet */
if (nfs_server_capable(hdr->inode, NFS_CAP_READ_PLUS) && !hdr->ds_clp) {
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ_PLUS];
- hdr->res.scratch = kmalloc(32, GFP_KERNEL);
- return hdr->res.scratch != NULL;
+ return nfs_read_alloc_scratch(hdr, READ_PLUS_SCRATCH_SIZE);
}
return false;
}
diff --git a/fs/nfs/read.c b/fs/nfs/read.c
index f71eeee67e20..7dc21a48e3e7 100644
--- a/fs/nfs/read.c
+++ b/fs/nfs/read.c
@@ -47,6 +47,8 @@ static struct nfs_pgio_header *nfs_readhdr_alloc(void)
static void nfs_readhdr_free(struct nfs_pgio_header *rhdr)
{
+ if (rhdr->res.scratch != NULL)
+ kfree(rhdr->res.scratch);
kmem_cache_free(nfs_rdata_cachep, rhdr);
}
@@ -108,6 +110,14 @@ void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
}
EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
+bool nfs_read_alloc_scratch(struct nfs_pgio_header *hdr, size_t size)
+{
+ WARN_ON(hdr->res.scratch != NULL);
+ hdr->res.scratch = kmalloc(size, GFP_KERNEL);
+ return hdr->res.scratch != NULL;
+}
+EXPORT_SYMBOL_GPL(nfs_read_alloc_scratch);
+
static void nfs_readpage_release(struct nfs_page *req, int error)
{
struct folio *folio = nfs_page_to_folio(req);
--
2.41.0
From: Anna Schumaker <[email protected]>
Now that the remaining issues have been worked out, including some
unexpected 32 bit issues, we can safely enable the feature by default.
Signed-off-by: Anna Schumaker <[email protected]>
---
fs/nfs/Kconfig | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig
index b6fc169be1b1..7df2503cef6c 100644
--- a/fs/nfs/Kconfig
+++ b/fs/nfs/Kconfig
@@ -209,8 +209,6 @@ config NFS_DISABLE_UDP_SUPPORT
config NFS_V4_2_READ_PLUS
bool "NFS: Enable support for the NFSv4.2 READ_PLUS operation"
depends on NFS_V4_2
- default n
+ default y
help
- This is intended for developers only. The READ_PLUS operation has
- been shown to have issues under specific conditions and should not
- be used in production.
+ Choose Y here to enable use of the NFS v4.2 READ_PLUS operation.
--
2.41.0
From: Anna Schumaker <[email protected]>
If the pages are in HIGHMEM then we need to make sure they're mapped
before trying to read data off of them, otherwise we could end up with a
NULL pointer dereference.
The downside to this is that we need an extra cleanup step at the end of
decode to kunmap() the last page. I introduced an xdr_finish_decode()
function to do this. Right now this function only calls the
unmap_current_page() function, but other generic cleanup steps could be
added in the future if we come across anything else.
Reported-by: Krzysztof Kozlowski <[email protected]>
Signed-off-by: Anna Schumaker <[email protected]>
---
include/linux/sunrpc/xdr.h | 3 +++
net/sunrpc/clnt.c | 1 +
net/sunrpc/svc.c | 2 ++
net/sunrpc/xdr.c | 27 ++++++++++++++++++++++++++-
4 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h
index 6bffd10b7a33..b75b3d416c5c 100644
--- a/include/linux/sunrpc/xdr.h
+++ b/include/linux/sunrpc/xdr.h
@@ -228,6 +228,7 @@ struct xdr_stream {
struct kvec *iov; /* pointer to the current kvec */
struct kvec scratch; /* Scratch buffer */
struct page **page_ptr; /* pointer to the current page */
+ void *page_kaddr; /* kmapped address of the current page */
unsigned int nwords; /* Remaining decode buffer length */
struct rpc_rqst *rqst; /* For debugging */
@@ -253,12 +254,14 @@ extern void xdr_truncate_decode(struct xdr_stream *xdr, size_t len);
extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen);
extern void xdr_write_pages(struct xdr_stream *xdr, struct page **pages,
unsigned int base, unsigned int len);
+extern void xdr_stream_unmap_current_page(struct xdr_stream *xdr);
extern unsigned int xdr_stream_pos(const struct xdr_stream *xdr);
extern unsigned int xdr_page_pos(const struct xdr_stream *xdr);
extern void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf,
__be32 *p, struct rpc_rqst *rqst);
extern void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
struct page **pages, unsigned int len);
+extern void xdr_finish_decode(struct xdr_stream *xdr);
extern __be32 *xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes);
extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len);
extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len);
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index d7c697af3762..ca2c6efe19c9 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -2602,6 +2602,7 @@ call_decode(struct rpc_task *task)
case 0:
task->tk_action = rpc_exit_task;
task->tk_status = rpcauth_unwrap_resp(task, &xdr);
+ xdr_finish_decode(&xdr);
return;
case -EAGAIN:
task->tk_status = 0;
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 587811a002c9..a864414ce811 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1370,6 +1370,8 @@ svc_process_common(struct svc_rqst *rqstp)
rc = process.dispatch(rqstp);
if (procp->pc_release)
procp->pc_release(rqstp);
+ xdr_finish_decode(xdr);
+
if (!rc)
goto dropit;
if (rqstp->rq_auth_stat != rpc_auth_ok)
diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index 94bddd1dd1d7..d318701a904a 100644
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -1306,6 +1306,14 @@ static unsigned int xdr_set_tail_base(struct xdr_stream *xdr,
return xdr_set_iov(xdr, buf->tail, base, len);
}
+void xdr_stream_unmap_current_page(struct xdr_stream *xdr)
+{
+ if (xdr->page_kaddr) {
+ kunmap_local(xdr->page_kaddr);
+ xdr->page_kaddr = NULL;
+ }
+}
+
static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
unsigned int base, unsigned int len)
{
@@ -1323,12 +1331,18 @@ static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
if (len > maxlen)
len = maxlen;
+ xdr_stream_unmap_current_page(xdr);
xdr_stream_page_set_pos(xdr, base);
base += xdr->buf->page_base;
pgnr = base >> PAGE_SHIFT;
xdr->page_ptr = &xdr->buf->pages[pgnr];
- kaddr = page_address(*xdr->page_ptr);
+
+ if (PageHighMem(*xdr->page_ptr)) {
+ xdr->page_kaddr = kmap_local_page(*xdr->page_ptr);
+ kaddr = xdr->page_kaddr;
+ } else
+ kaddr = page_address(*xdr->page_ptr);
pgoff = base & ~PAGE_MASK;
xdr->p = (__be32*)(kaddr + pgoff);
@@ -1382,6 +1396,7 @@ void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
struct rpc_rqst *rqst)
{
xdr->buf = buf;
+ xdr->page_kaddr = NULL;
xdr_reset_scratch_buffer(xdr);
xdr->nwords = XDR_QUADLEN(buf->len);
if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&
@@ -1414,6 +1429,16 @@ void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
}
EXPORT_SYMBOL_GPL(xdr_init_decode_pages);
+/**
+ * xdr_finish_decode - Clean up the xdr_stream after decoding data.
+ * @xdr: pointer to xdr_stream struct
+ */
+void xdr_finish_decode(struct xdr_stream *xdr)
+{
+ xdr_stream_unmap_current_page(xdr);
+}
+EXPORT_SYMBOL(xdr_finish_decode);
+
static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
{
unsigned int nwords = XDR_QUADLEN(nbytes);
--
2.41.0
On Fri, Jul 21, 2023 at 04:39:52PM -0400, Anna Schumaker wrote:
> From: Anna Schumaker <[email protected]>
>
> If the pages are in HIGHMEM then we need to make sure they're mapped
> before trying to read data off of them, otherwise we could end up with a
> NULL pointer dereference.
>
> The downside to this is that we need an extra cleanup step at the end of
> decode to kunmap() the last page. I introduced an xdr_finish_decode()
> function to do this. Right now this function only calls the
> unmap_current_page() function, but other generic cleanup steps could be
> added in the future if we come across anything else.
>
> Reported-by: Krzysztof Kozlowski <[email protected]>
> Signed-off-by: Anna Schumaker <[email protected]>
> ---
> include/linux/sunrpc/xdr.h | 3 +++
> net/sunrpc/clnt.c | 1 +
> net/sunrpc/svc.c | 2 ++
> net/sunrpc/xdr.c | 27 ++++++++++++++++++++++++++-
> 4 files changed, 32 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h
> index 6bffd10b7a33..b75b3d416c5c 100644
> --- a/include/linux/sunrpc/xdr.h
> +++ b/include/linux/sunrpc/xdr.h
> @@ -228,6 +228,7 @@ struct xdr_stream {
> struct kvec *iov; /* pointer to the current kvec */
> struct kvec scratch; /* Scratch buffer */
> struct page **page_ptr; /* pointer to the current page */
> + void *page_kaddr; /* kmapped address of the current page */
> unsigned int nwords; /* Remaining decode buffer length */
>
> struct rpc_rqst *rqst; /* For debugging */
> @@ -253,12 +254,14 @@ extern void xdr_truncate_decode(struct xdr_stream *xdr, size_t len);
> extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen);
> extern void xdr_write_pages(struct xdr_stream *xdr, struct page **pages,
> unsigned int base, unsigned int len);
> +extern void xdr_stream_unmap_current_page(struct xdr_stream *xdr);
Nit: I don't think this function needs to be public any more. The
usual convention is to make helpers like this static until they are
needed somewhere outside the source file where they are defined.
The current definition (below) of xdr_stream_unmap_current_page()
has neither an EXPORT_SYMBOL nor a kdoc comment, so seems like an
easy change to make it static.
I don't see other issues, so you could just make this change as
you apply it rather than posting a v7, but it's your nickel.
> extern unsigned int xdr_stream_pos(const struct xdr_stream *xdr);
> extern unsigned int xdr_page_pos(const struct xdr_stream *xdr);
> extern void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf,
> __be32 *p, struct rpc_rqst *rqst);
> extern void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
> struct page **pages, unsigned int len);
> +extern void xdr_finish_decode(struct xdr_stream *xdr);
> extern __be32 *xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes);
> extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len);
> extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len);
> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> index d7c697af3762..ca2c6efe19c9 100644
> --- a/net/sunrpc/clnt.c
> +++ b/net/sunrpc/clnt.c
> @@ -2602,6 +2602,7 @@ call_decode(struct rpc_task *task)
> case 0:
> task->tk_action = rpc_exit_task;
> task->tk_status = rpcauth_unwrap_resp(task, &xdr);
> + xdr_finish_decode(&xdr);
> return;
> case -EAGAIN:
> task->tk_status = 0;
> diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
> index 587811a002c9..a864414ce811 100644
> --- a/net/sunrpc/svc.c
> +++ b/net/sunrpc/svc.c
> @@ -1370,6 +1370,8 @@ svc_process_common(struct svc_rqst *rqstp)
> rc = process.dispatch(rqstp);
> if (procp->pc_release)
> procp->pc_release(rqstp);
> + xdr_finish_decode(xdr);
> +
> if (!rc)
> goto dropit;
> if (rqstp->rq_auth_stat != rpc_auth_ok)
> diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
> index 94bddd1dd1d7..d318701a904a 100644
> --- a/net/sunrpc/xdr.c
> +++ b/net/sunrpc/xdr.c
> @@ -1306,6 +1306,14 @@ static unsigned int xdr_set_tail_base(struct xdr_stream *xdr,
> return xdr_set_iov(xdr, buf->tail, base, len);
> }
>
> +void xdr_stream_unmap_current_page(struct xdr_stream *xdr)
> +{
> + if (xdr->page_kaddr) {
> + kunmap_local(xdr->page_kaddr);
> + xdr->page_kaddr = NULL;
> + }
> +}
> +
> static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
> unsigned int base, unsigned int len)
> {
> @@ -1323,12 +1331,18 @@ static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
> if (len > maxlen)
> len = maxlen;
>
> + xdr_stream_unmap_current_page(xdr);
> xdr_stream_page_set_pos(xdr, base);
> base += xdr->buf->page_base;
>
> pgnr = base >> PAGE_SHIFT;
> xdr->page_ptr = &xdr->buf->pages[pgnr];
> - kaddr = page_address(*xdr->page_ptr);
> +
> + if (PageHighMem(*xdr->page_ptr)) {
> + xdr->page_kaddr = kmap_local_page(*xdr->page_ptr);
> + kaddr = xdr->page_kaddr;
> + } else
> + kaddr = page_address(*xdr->page_ptr);
>
> pgoff = base & ~PAGE_MASK;
> xdr->p = (__be32*)(kaddr + pgoff);
> @@ -1382,6 +1396,7 @@ void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
> struct rpc_rqst *rqst)
> {
> xdr->buf = buf;
> + xdr->page_kaddr = NULL;
> xdr_reset_scratch_buffer(xdr);
> xdr->nwords = XDR_QUADLEN(buf->len);
> if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&
> @@ -1414,6 +1429,16 @@ void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
> }
> EXPORT_SYMBOL_GPL(xdr_init_decode_pages);
>
> +/**
> + * xdr_finish_decode - Clean up the xdr_stream after decoding data.
> + * @xdr: pointer to xdr_stream struct
> + */
> +void xdr_finish_decode(struct xdr_stream *xdr)
> +{
> + xdr_stream_unmap_current_page(xdr);
> +}
> +EXPORT_SYMBOL(xdr_finish_decode);
> +
> static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
> {
> unsigned int nwords = XDR_QUADLEN(nbytes);
> --
> 2.41.0
>
--
Chuck Lever