Hi folks,
Recently, there are two new cases so that we need to add a preliminary
sub-page block support for compressed files;
- As Android folks requested, Android ecosystem itself is now switching
to 16k page size for their arm64 devices. They needs an option of
4k-block image compatibility on their new 16k devices;
- Some arm64 cloud servers use 64k page size for their optimized
workloads, but 4k-block EROFS container images need to be parsed too.
So this patchset mainly addresses the requirements above with a very
very simple approach as a start: just allocate short-lived temporary
buffers all the time to keep compressed data if sub-page blocks are
identified. In other words, no inplace/cache decompression for
the preliminary support.
This patchset survives EROFS stress test on my own testfarms.
Thanks,
Gao Xiang
Gao Xiang (5):
erofs: support I/O submission for sub-page compressed blocks
erofs: record `pclustersize` in bytes instead of pages
erofs: fix up compacted indexes for block size < 4096
erofs: refine z_erofs_transform_plain() for sub-page block support
erofs: enable sub-page compressed block support
fs/erofs/decompressor.c | 81 +++++++++------
fs/erofs/inode.c | 6 +-
fs/erofs/zdata.c | 224 ++++++++++++++++++----------------------
fs/erofs/zmap.c | 32 +++---
4 files changed, 169 insertions(+), 174 deletions(-)
--
2.39.3
Previously, the block size always equaled to PAGE_SIZE, therefore
`lclusterbits` couldn't be less than 12.
Since sub-page compressed blocks are now considered, `lobits` for
a lcluster in each pack cannot always be `lclusterbits` as before.
Otherwise, there is no enough room for the special value
`Z_EROFS_LI_D0_CBLKCNT`.
To support smaller block sizes, `lobits` for each compacted lcluster is
now calculated as:
lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1)
Signed-off-by: Gao Xiang <[email protected]>
---
fs/erofs/zmap.c | 32 ++++++++++++++------------------
1 file changed, 14 insertions(+), 18 deletions(-)
diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c
index 7b55111fd533..9753875e41cb 100644
--- a/fs/erofs/zmap.c
+++ b/fs/erofs/zmap.c
@@ -82,29 +82,26 @@ static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m,
}
static unsigned int decode_compactedbits(unsigned int lobits,
- unsigned int lomask,
u8 *in, unsigned int pos, u8 *type)
{
const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
- const unsigned int lo = v & lomask;
+ const unsigned int lo = v & ((1 << lobits) - 1);
*type = (v >> lobits) & 3;
return lo;
}
-static int get_compacted_la_distance(unsigned int lclusterbits,
+static int get_compacted_la_distance(unsigned int lobits,
unsigned int encodebits,
unsigned int vcnt, u8 *in, int i)
{
- const unsigned int lomask = (1 << lclusterbits) - 1;
unsigned int lo, d1 = 0;
u8 type;
DBG_BUGON(i >= vcnt);
do {
- lo = decode_compactedbits(lclusterbits, lomask,
- in, encodebits * i, &type);
+ lo = decode_compactedbits(lobits, in, encodebits * i, &type);
if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
return d1;
@@ -123,15 +120,14 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
{
struct erofs_inode *const vi = EROFS_I(m->inode);
const unsigned int lclusterbits = vi->z_logical_clusterbits;
- const unsigned int lomask = (1 << lclusterbits) - 1;
- unsigned int vcnt, base, lo, encodebits, nblk, eofs;
+ unsigned int vcnt, base, lo, lobits, encodebits, nblk, eofs;
int i;
u8 *in, type;
bool big_pcluster;
if (1 << amortizedshift == 4 && lclusterbits <= 14)
vcnt = 2;
- else if (1 << amortizedshift == 2 && lclusterbits == 12)
+ else if (1 << amortizedshift == 2 && lclusterbits <= 12)
vcnt = 16;
else
return -EOPNOTSUPP;
@@ -140,6 +136,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
(vcnt << amortizedshift);
big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
+ lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U);
encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
eofs = erofs_blkoff(m->inode->i_sb, pos);
base = round_down(eofs, vcnt << amortizedshift);
@@ -147,15 +144,14 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
i = (eofs - base) >> amortizedshift;
- lo = decode_compactedbits(lclusterbits, lomask,
- in, encodebits * i, &type);
+ lo = decode_compactedbits(lobits, in, encodebits * i, &type);
m->type = type;
if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
m->clusterofs = 1 << lclusterbits;
/* figure out lookahead_distance: delta[1] if needed */
if (lookahead)
- m->delta[1] = get_compacted_la_distance(lclusterbits,
+ m->delta[1] = get_compacted_la_distance(lobits,
encodebits, vcnt, in, i);
if (lo & Z_EROFS_LI_D0_CBLKCNT) {
if (!big_pcluster) {
@@ -174,8 +170,8 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
* of which lo saves delta[1] rather than delta[0].
* Hence, get delta[0] by the previous lcluster indirectly.
*/
- lo = decode_compactedbits(lclusterbits, lomask,
- in, encodebits * (i - 1), &type);
+ lo = decode_compactedbits(lobits, in,
+ encodebits * (i - 1), &type);
if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
lo = 0;
else if (lo & Z_EROFS_LI_D0_CBLKCNT)
@@ -190,8 +186,8 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
nblk = 1;
while (i > 0) {
--i;
- lo = decode_compactedbits(lclusterbits, lomask,
- in, encodebits * i, &type);
+ lo = decode_compactedbits(lobits, in,
+ encodebits * i, &type);
if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD)
i -= lo;
@@ -202,8 +198,8 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
nblk = 0;
while (i > 0) {
--i;
- lo = decode_compactedbits(lclusterbits, lomask,
- in, encodebits * i, &type);
+ lo = decode_compactedbits(lobits, in,
+ encodebits * i, &type);
if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
if (lo & Z_EROFS_LI_D0_CBLKCNT) {
--i;
--
2.39.3
On Wed, 6 Dec 2023 17:10:52 +0800
Gao Xiang <[email protected]> wrote:
> Hi folks,
>
> Recently, there are two new cases so that we need to add a preliminary
> sub-page block support for compressed files;
>
> - As Android folks requested, Android ecosystem itself is now switching
> to 16k page size for their arm64 devices. They needs an option of
> 4k-block image compatibility on their new 16k devices;
>
> - Some arm64 cloud servers use 64k page size for their optimized
> workloads, but 4k-block EROFS container images need to be parsed too.
>
> So this patchset mainly addresses the requirements above with a very
> very simple approach as a start: just allocate short-lived temporary
> buffers all the time to keep compressed data if sub-page blocks are
> identified. In other words, no inplace/cache decompression for
> the preliminary support.
>
> This patchset survives EROFS stress test on my own testfarms.
>
> Thanks,
> Gao Xiang
>
> Gao Xiang (5):
> erofs: support I/O submission for sub-page compressed blocks
> erofs: record `pclustersize` in bytes instead of pages
> erofs: fix up compacted indexes for block size < 4096
> erofs: refine z_erofs_transform_plain() for sub-page block support
> erofs: enable sub-page compressed block support
>
> fs/erofs/decompressor.c | 81 +++++++++------
> fs/erofs/inode.c | 6 +-
> fs/erofs/zdata.c | 224 ++++++++++++++++++----------------------
> fs/erofs/zmap.c | 32 +++---
> 4 files changed, 169 insertions(+), 174 deletions(-)
>
Reviewed-by: Yue Hu <[email protected]>
On 2023/12/6 17:10, Gao Xiang wrote:
> Hi folks,
>
> Recently, there are two new cases so that we need to add a preliminary
> sub-page block support for compressed files;
>
> - As Android folks requested, Android ecosystem itself is now switching
> to 16k page size for their arm64 devices. They needs an option of
> 4k-block image compatibility on their new 16k devices;
>
> - Some arm64 cloud servers use 64k page size for their optimized
> workloads, but 4k-block EROFS container images need to be parsed too.
>
> So this patchset mainly addresses the requirements above with a very
> very simple approach as a start: just allocate short-lived temporary
> buffers all the time to keep compressed data if sub-page blocks are
> identified. In other words, no inplace/cache decompression for
> the preliminary support.
>
> This patchset survives EROFS stress test on my own testfarms.
Looks good to me, feel free to add:
Reviewed-by: Chao Yu <[email protected]>
Thanks,
>
> Thanks,
> Gao Xiang
>
> Gao Xiang (5):
> erofs: support I/O submission for sub-page compressed blocks
> erofs: record `pclustersize` in bytes instead of pages
> erofs: fix up compacted indexes for block size < 4096
> erofs: refine z_erofs_transform_plain() for sub-page block support
> erofs: enable sub-page compressed block support
>
> fs/erofs/decompressor.c | 81 +++++++++------
> fs/erofs/inode.c | 6 +-
> fs/erofs/zdata.c | 224 ++++++++++++++++++----------------------
> fs/erofs/zmap.c | 32 +++---
> 4 files changed, 169 insertions(+), 174 deletions(-)
>