Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751869AbdGaBuW (ORCPT ); Sun, 30 Jul 2017 21:50:22 -0400 Received: from mail-vk0-f67.google.com ([209.85.213.67]:32822 "EHLO mail-vk0-f67.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751688AbdGaBuU (ORCPT ); Sun, 30 Jul 2017 21:50:20 -0400 MIME-Version: 1.0 In-Reply-To: <20170720212743.3351201-5-terrelln@fb.com> References: <20170720212743.3351201-1-terrelln@fb.com> <20170720212743.3351201-5-terrelln@fb.com> From: Phillip Lougher Date: Mon, 31 Jul 2017 02:50:18 +0100 Message-ID: Subject: Re: [PATCH v3 4/4] squashfs: Add zstd support To: Nick Terrell Cc: "Austin S . Hemmelgarn" , Herbert Xu , kernel-team@fb.com, Chris Mason , Yann Collet , Adam Borowski , David Sterba , SquashFS developers , linux-btrfs@vger.kernel.org, LKML , Sean Purcell Content-Type: text/plain; charset="UTF-8" Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 10660 Lines: 300 On Thu, Jul 20, 2017 at 10:27 PM, Nick Terrell wrote: > Add zstd compression and decompression support to SquashFS. zstd is a > great fit for SquashFS because it can compress at ratios approaching xz, > while decompressing twice as fast as zlib. For SquashFS in particular, > it can decompress as fast as lzo and lz4. It also has the flexibility > to turn down the compression ratio for faster compression times. Hi Nick, This patch (and none of the previous versions) is showing up on squashfs-devel@lists.sourceforge.net. I also think you should have emailed me directly as a courtesy, as I'm the Squashfs author and maintainer. > | Method | Ratio | Compression MB/s | Decompression MB/s | > |----------------|-------|------------------|--------------------| > | gzip | 2.92 | 15 | 128 | > | lzo | 2.64 | 9.5 | 217 | > | lz4 | 2.12 | 94 | 218 | > | xz | 3.43 | 5.5 | 35 | > | xz 256 KB | 3.53 | 5.4 | 40 | > | zstd 1 | 2.71 | 96 | 210 | > | zstd 5 | 2.93 | 69 | 198 | > | zstd 10 | 3.01 | 41 | 225 | > | zstd 15 | 3.13 | 11.4 | 224 | > | zstd 16 256 KB | 3.24 | 8.1 | 210 | Those numbers look good to me. > > This patch was written by Sean Purcell , but I will be > taking over the submission process. > > [1] http://releases.ubuntu.com/16.10/ > [2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh I can't find your patch that adds zstd to the user-land Squashfs-tools. That would be handy to do any testing :-) Phillip > > zstd source repository: https://github.com/facebook/zstd > > Cc: Sean Purcell > Signed-off-by: Nick Terrell > --- > fs/squashfs/Kconfig | 14 +++++ > fs/squashfs/Makefile | 1 + > fs/squashfs/decompressor.c | 7 +++ > fs/squashfs/decompressor.h | 4 ++ > fs/squashfs/squashfs_fs.h | 1 + > fs/squashfs/zstd_wrapper.c | 150 +++++++++++++++++++++++++++++++++++++++++++++ > 6 files changed, 177 insertions(+) > create mode 100644 fs/squashfs/zstd_wrapper.c > > diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig > index ffb093e..1adb334 100644 > --- a/fs/squashfs/Kconfig > +++ b/fs/squashfs/Kconfig > @@ -165,6 +165,20 @@ config SQUASHFS_XZ > > If unsure, say N. > > +config SQUASHFS_ZSTD > + bool "Include support for ZSTD compressed file systems" > + depends on SQUASHFS > + select ZSTD_DECOMPRESS > + help > + Saying Y here includes support for reading Squashfs file systems > + compressed with ZSTD compression. ZSTD gives better compression than > + the default ZLIB compression, while using less CPU. > + > + ZSTD is not the standard compression used in Squashfs and so most > + file systems will be readable without selecting this option. > + > + If unsure, say N. > + > config SQUASHFS_4K_DEVBLK_SIZE > bool "Use 4K device block size?" > depends on SQUASHFS > diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile > index 246a6f3..6655631 100644 > --- a/fs/squashfs/Makefile > +++ b/fs/squashfs/Makefile > @@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o > squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o > squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o > squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o > +squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o > diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c > index d2bc136..8366398 100644 > --- a/fs/squashfs/decompressor.c > +++ b/fs/squashfs/decompressor.c > @@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = { > }; > #endif > > +#ifndef CONFIG_SQUASHFS_ZSTD > +static const struct squashfs_decompressor squashfs_zstd_comp_ops = { > + NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0 > +}; > +#endif > + > static const struct squashfs_decompressor squashfs_unknown_comp_ops = { > NULL, NULL, NULL, NULL, 0, "unknown", 0 > }; > @@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = { > &squashfs_lzo_comp_ops, > &squashfs_xz_comp_ops, > &squashfs_lzma_unsupported_comp_ops, > + &squashfs_zstd_comp_ops, > &squashfs_unknown_comp_ops > }; > > diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h > index a25713c..0f5a8e4 100644 > --- a/fs/squashfs/decompressor.h > +++ b/fs/squashfs/decompressor.h > @@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops; > extern const struct squashfs_decompressor squashfs_zlib_comp_ops; > #endif > > +#ifdef CONFIG_SQUASHFS_ZSTD > +extern const struct squashfs_decompressor squashfs_zstd_comp_ops; > +#endif > + > #endif > diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h > index 506f4ba..24d12fd 100644 > --- a/fs/squashfs/squashfs_fs.h > +++ b/fs/squashfs/squashfs_fs.h > @@ -241,6 +241,7 @@ struct meta_index { > #define LZO_COMPRESSION 3 > #define XZ_COMPRESSION 4 > #define LZ4_COMPRESSION 5 > +#define ZSTD_COMPRESSION 6 > > struct squashfs_super_block { > __le32 s_magic; > diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c > new file mode 100644 > index 0000000..8cb7c76 > --- /dev/null > +++ b/fs/squashfs/zstd_wrapper.c > @@ -0,0 +1,150 @@ > +/* > + * Squashfs - a compressed read only filesystem for Linux > + * > + * Copyright (c) 2016-present, Facebook, Inc. > + * All rights reserved. > + * > + * This program is free software; you can redistribute it and/or > + * modify it under the terms of the GNU General Public License > + * as published by the Free Software Foundation; either version 2, > + * or (at your option) any later version. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, write to the Free Software > + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. > + * > + * zstd_wrapper.c > + */ > + > +#include > +#include > +#include > +#include > +#include > + > +#include "squashfs_fs.h" > +#include "squashfs_fs_sb.h" > +#include "squashfs.h" > +#include "decompressor.h" > +#include "page_actor.h" > + > +struct workspace { > + void *mem; > + size_t mem_size; > +}; > + > +static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) > +{ > + struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL); > + if (wksp == NULL) > + goto failed; > + wksp->mem_size = ZSTD_DStreamWorkspaceBound(max_t(size_t, > + msblk->block_size, SQUASHFS_METADATA_SIZE)); > + wksp->mem = vmalloc(wksp->mem_size); > + if (wksp->mem == NULL) > + goto failed; > + > + return wksp; > + > +failed: > + ERROR("Failed to allocate zstd workspace\n"); > + kfree(wksp); > + return ERR_PTR(-ENOMEM); > +} > + > + > +static void zstd_free(void *strm) > +{ > + struct workspace *wksp = strm; > + > + if (wksp) > + vfree(wksp->mem); > + kfree(wksp); > +} > + > + > +static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, > + struct buffer_head **bh, int b, int offset, int length, > + struct squashfs_page_actor *output) > +{ > + struct workspace *wksp = strm; > + ZSTD_DStream *stream; > + size_t total_out = 0; > + size_t zstd_err; > + int k = 0; > + ZSTD_inBuffer in_buf = { NULL, 0, 0 }; > + ZSTD_outBuffer out_buf = { NULL, 0, 0 }; > + > + stream = ZSTD_initDStream(wksp->mem_size, wksp->mem, wksp->mem_size); > + > + if (!stream) { > + ERROR("Failed to initialize zstd decompressor\n"); > + goto out; > + } > + > + out_buf.size = PAGE_SIZE; > + out_buf.dst = squashfs_first_page(output); > + > + do { > + if (in_buf.pos == in_buf.size && k < b) { > + int avail = min(length, msblk->devblksize - offset); > + length -= avail; > + in_buf.src = bh[k]->b_data + offset; > + in_buf.size = avail; > + in_buf.pos = 0; > + offset = 0; > + } > + > + if (out_buf.pos == out_buf.size) { > + out_buf.dst = squashfs_next_page(output); > + if (out_buf.dst == NULL) { > + /* shouldn't run out of pages before stream is > + * done */ > + squashfs_finish_page(output); > + goto out; > + } > + out_buf.pos = 0; > + out_buf.size = PAGE_SIZE; > + } > + > + total_out -= out_buf.pos; > + zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); > + total_out += out_buf.pos; /* add the additional data produced */ > + > + if (in_buf.pos == in_buf.size && k < b) > + put_bh(bh[k++]); > + } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); > + > + squashfs_finish_page(output); > + > + if (ZSTD_isError(zstd_err)) { > + ERROR("zstd decompression error: %d\n", > + (int)ZSTD_getErrorCode(zstd_err)); > + goto out; > + } > + > + if (k < b) > + goto out; > + > + return (int)total_out; > + > +out: > + for (; k < b; k++) > + put_bh(bh[k]); > + > + return -EIO; > +} > + > +const struct squashfs_decompressor squashfs_zstd_comp_ops = { > + .init = zstd_init, > + .free = zstd_free, > + .decompress = zstd_uncompress, > + .id = ZSTD_COMPRESSION, > + .name = "zstd", > + .supported = 1 > +}; > -- > 2.9.3