XZ Embedded, the upstream project, switched from public domain to the
BSD Zero Clause License (0BSD). Now matching SPDX license identifiers
can be added.
The new ARM64 and RISC-V filters can be used by Squashfs.
Account for the default threading change made in the xz command line
tool version 5.6.0. Tweak kernel compression options for archs that
support XZ compressed kernel.
Documentation was revised. There are minor cleanups too.
Lasse Collin (11):
MAINTAINERS: Add XZ Embedded maintainers
LICENSES: Add 0BSD license text
xz: Switch from public domain to BSD Zero Clause License (0BSD)
xz: Documentation/staging/xz.rst: Revise thoroughly
xz: Fix comments and coding style
xz: Cleanup CRC32 edits from 2018
xz: Optimize for-loop conditions in the BCJ decoders
xz: Add ARM64 BCJ filter
xz: Add RISC-V BCJ filter
xz: Use 128 MiB dictionary and force single-threaded mode
xz: Adjust arch-specific options for better kernel compression
Documentation/staging/xz.rst | 130 ++++++++---------------
LICENSES/deprecated/0BSD | 23 ++++
MAINTAINERS | 14 +++
include/linux/decompress/unxz.h | 5 +-
include/linux/xz.h | 5 +-
init/Kconfig | 5 +-
lib/decompress_unxz.c | 39 ++++---
lib/xz/Kconfig | 13 ++-
lib/xz/xz_crc32.c | 7 +-
lib/xz/xz_dec_bcj.c | 183 ++++++++++++++++++++++++++++++--
lib/xz/xz_dec_lzma2.c | 5 +-
lib/xz/xz_dec_stream.c | 5 +-
lib/xz/xz_dec_syms.c | 16 +--
lib/xz/xz_dec_test.c | 12 +--
lib/xz/xz_lzma2.h | 5 +-
lib/xz/xz_private.h | 20 ++--
lib/xz/xz_stream.h | 7 +-
scripts/Makefile.lib | 13 ++-
scripts/xz_wrap.sh | 157 +++++++++++++++++++++++++--
19 files changed, 487 insertions(+), 177 deletions(-)
create mode 100644 LICENSES/deprecated/0BSD
--
2.44.0
The license text was copied from:
https://spdx.org/licenses/0BSD.html
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
Notes:
0BSD is the ISC license without the requirements to preserve copyright
and license notices.
LICENSES/deprecated/0BSD | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 LICENSES/deprecated/0BSD
diff --git a/LICENSES/deprecated/0BSD b/LICENSES/deprecated/0BSD
new file mode 100644
index 000000000000..e4b95b749966
--- /dev/null
+++ b/LICENSES/deprecated/0BSD
@@ -0,0 +1,23 @@
+Valid-License-Identifier: 0BSD
+SPDX-URL: https://spdx.org/licenses/0BSD.html
+Usage-Guide:
+ To use the BSD Zero Clause License put the following SPDX tag/value
+ pair into a comment according to the placement guidelines in the
+ licensing rules documentation:
+ SPDX-License-Identifier: 0BSD
+License-Text:
+
+BSD Zero Clause License
+
+Copyright (c) <year> <copyright holders>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
--
2.44.0
Compilers cannot optimize the addition "i + 4" away since theoretically
it could overflow.
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
lib/xz/xz_dec_bcj.c | 23 ++++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
index ab9237ed6db8..e0b4bf4999c0 100644
--- a/lib/xz/xz_dec_bcj.c
+++ b/lib/xz/xz_dec_bcj.c
@@ -161,7 +161,9 @@ static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
size_t i;
uint32_t instr;
- for (i = 0; i + 4 <= size; i += 4) {
+ size &= ~(size_t)3;
+
+ for (i = 0; i < size; i += 4) {
instr = get_unaligned_be32(buf + i);
if ((instr & 0xFC000003) == 0x48000001) {
instr &= 0x03FFFFFC;
@@ -218,7 +220,9 @@ static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
/* Instruction normalized with bit_res for easier manipulation */
uint64_t norm;
- for (i = 0; i + 16 <= size; i += 16) {
+ size &= ~(size_t)15;
+
+ for (i = 0; i < size; i += 16) {
mask = branch_table[buf[i] & 0x1F];
for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) {
if (((mask >> slot) & 1) == 0)
@@ -266,7 +270,9 @@ static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
size_t i;
uint32_t addr;
- for (i = 0; i + 4 <= size; i += 4) {
+ size &= ~(size_t)3;
+
+ for (i = 0; i < size; i += 4) {
if (buf[i + 3] == 0xEB) {
addr = (uint32_t)buf[i] | ((uint32_t)buf[i + 1] << 8)
| ((uint32_t)buf[i + 2] << 16);
@@ -289,7 +295,12 @@ static size_t bcj_armthumb(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
size_t i;
uint32_t addr;
- for (i = 0; i + 4 <= size; i += 2) {
+ if (size < 4)
+ return 0;
+
+ size -= 4;
+
+ for (i = 0; i <= size; i += 2) {
if ((buf[i + 1] & 0xF8) == 0xF0
&& (buf[i + 3] & 0xF8) == 0xF8) {
addr = (((uint32_t)buf[i + 1] & 0x07) << 19)
@@ -317,7 +328,9 @@ static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
size_t i;
uint32_t instr;
- for (i = 0; i + 4 <= size; i += 4) {
+ size &= ~(size_t)3;
+
+ for (i = 0; i < size; i += 4) {
instr = get_unaligned_be32(buf + i);
if ((instr >> 22) == 0x100 || (instr >> 22) == 0x1FF) {
instr <<= 2;
--
2.44.0
Remove the public domain notices and add SPDX license identifiers.
Change MODULE_LICENSE from "GPL" to "Dual BSD/GPL" because 0BSD should
count as a BSD license variant here.
The switch to 0BSD was done in the upstream XZ Embedded project because
public domain has (real or perceived) legal issues in some jurisdictions.
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
include/linux/decompress/unxz.h | 5 ++---
include/linux/xz.h | 5 ++---
lib/decompress_unxz.c | 5 ++---
lib/xz/xz_crc32.c | 5 ++---
lib/xz/xz_dec_bcj.c | 5 ++---
lib/xz/xz_dec_lzma2.c | 5 ++---
lib/xz/xz_dec_stream.c | 5 ++---
lib/xz/xz_dec_syms.c | 12 +++---------
lib/xz/xz_dec_test.c | 12 +++---------
lib/xz/xz_lzma2.h | 5 ++---
lib/xz/xz_private.h | 5 ++---
lib/xz/xz_stream.h | 5 ++---
scripts/xz_wrap.sh | 5 +----
13 files changed, 27 insertions(+), 52 deletions(-)
diff --git a/include/linux/decompress/unxz.h b/include/linux/decompress/unxz.h
index f764e2a7201e..3dd2658a9dab 100644
--- a/include/linux/decompress/unxz.h
+++ b/include/linux/decompress/unxz.h
@@ -1,10 +1,9 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#ifndef DECOMPRESS_UNXZ_H
diff --git a/include/linux/xz.h b/include/linux/xz.h
index 7285ca5d56e9..5728d57aecc0 100644
--- a/include/linux/xz.h
+++ b/include/linux/xz.h
@@ -1,11 +1,10 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* XZ decompressor
*
* Authors: Lasse Collin <[email protected]>
* Igor Pavlov <https://7-zip.org/>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#ifndef XZ_H
diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
index 842894158944..34bb7efc0412 100644
--- a/lib/decompress_unxz.c
+++ b/lib/decompress_unxz.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
/*
diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c
index 88a2c35e1b59..30b8a27110b1 100644
--- a/lib/xz/xz_crc32.c
+++ b/lib/xz/xz_crc32.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* CRC32 using the polynomial from IEEE-802.3
*
* Authors: Lasse Collin <[email protected]>
* Igor Pavlov <https://7-zip.org/>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
/*
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
index ef449e97d1a1..ab9237ed6db8 100644
--- a/lib/xz/xz_dec_bcj.c
+++ b/lib/xz/xz_dec_bcj.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* Branch/Call/Jump (BCJ) filter decoders
*
* Authors: Lasse Collin <[email protected]>
* Igor Pavlov <https://7-zip.org/>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#include "xz_private.h"
diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c
index 27ce34520e78..613939f5dd6c 100644
--- a/lib/xz/xz_dec_lzma2.c
+++ b/lib/xz/xz_dec_lzma2.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* LZMA2 decoder
*
* Authors: Lasse Collin <[email protected]>
* Igor Pavlov <https://7-zip.org/>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#include "xz_private.h"
diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
index 683570b93a8c..0058406ccd17 100644
--- a/lib/xz/xz_dec_stream.c
+++ b/lib/xz/xz_dec_stream.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* .xz Stream decoder
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#include "xz_private.h"
diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c
index 61098c67a413..495d2cc2e6e8 100644
--- a/lib/xz/xz_dec_syms.c
+++ b/lib/xz/xz_dec_syms.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* XZ decoder module information
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#include <linux/module.h>
@@ -25,9 +24,4 @@ EXPORT_SYMBOL(xz_dec_microlzma_end);
MODULE_DESCRIPTION("XZ decompressor");
MODULE_VERSION("1.1");
MODULE_AUTHOR("Lasse Collin <[email protected]> and Igor Pavlov");
-
-/*
- * This code is in the public domain, but in Linux it's simplest to just
- * say it's GPL and consider the authors as the copyright holders.
- */
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/xz/xz_dec_test.c b/lib/xz/xz_dec_test.c
index da28a19d6c98..53d3600f2ddb 100644
--- a/lib/xz/xz_dec_test.c
+++ b/lib/xz/xz_dec_test.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: 0BSD
+
/*
* XZ decoder tester
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#include <linux/kernel.h>
@@ -212,9 +211,4 @@ module_exit(xz_dec_test_exit);
MODULE_DESCRIPTION("XZ decompressor tester");
MODULE_VERSION("1.0");
MODULE_AUTHOR("Lasse Collin <[email protected]>");
-
-/*
- * This code is in the public domain, but in Linux it's simplest to just
- * say it's GPL and consider the authors as the copyright holders.
- */
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h
index 92d852d4f87a..d2632b7dfb9c 100644
--- a/lib/xz/xz_lzma2.h
+++ b/lib/xz/xz_lzma2.h
@@ -1,11 +1,10 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* LZMA2 definitions
*
* Authors: Lasse Collin <[email protected]>
* Igor Pavlov <https://7-zip.org/>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#ifndef XZ_LZMA2_H
diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h
index bf1e94ec7873..2412a5d54801 100644
--- a/lib/xz/xz_private.h
+++ b/lib/xz/xz_private.h
@@ -1,10 +1,9 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* Private includes and definitions
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#ifndef XZ_PRIVATE_H
diff --git a/lib/xz/xz_stream.h b/lib/xz/xz_stream.h
index 430bb3a0d195..55f9f6f94b78 100644
--- a/lib/xz/xz_stream.h
+++ b/lib/xz/xz_stream.h
@@ -1,10 +1,9 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* Definitions for handling the .xz file format
*
* Author: Lasse Collin <[email protected]>
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#ifndef XZ_STREAM_H
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index d06baf626abe..bb760b721b2c 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -1,13 +1,10 @@
#!/bin/sh
+# SPDX-License-Identifier: 0BSD
#
# This is a wrapper for xz to compress the kernel image using appropriate
# compression options depending on the architecture.
#
# Author: Lasse Collin <[email protected]>
-#
-# This file has been put into the public domain.
-# You can do whatever you want with this file.
-#
BCJ=
LZMA2OPTS=
--
2.44.0
I have been the maintainer of the upstream project since I submitted
the code to Linux in 2010 but I forgot to add myself to MAINTAINERS.
Nowadays Jia Tan is the other maintainer.
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
MAINTAINERS | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 43b39956694a..35f4ee968626 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -24371,6 +24371,20 @@ S: Maintained
F: drivers/spi/spi-xtensa-xtfpga.c
F: sound/soc/xtensa/xtfpga-i2s.c
+XZ EMBEDDED
+M: Lasse Collin <[email protected]>
+M: Jia Tan <[email protected]>
+S: Maintained
+W: https://xz.tukaani.org/xz-embedded/
+B: https://github.com/tukaani-project/xz-embedded/issues
+C: irc://irc.libera.chat/tukaani
+F: Documentation/staging/xz.rst
+F: include/linux/decompress/unxz.h
+F: include/linux/xz.h
+F: lib/decompress_unxz.c
+F: lib/xz/
+F: scripts/xz_wrap.sh
+
YAM DRIVER FOR AX.25
M: Jean-Paul Roubelat <[email protected]>
L: [email protected]
--
2.44.0
Add SPDX license identifier.
Omit xz_dec_test info. That isn't relevant to developers of non-XZ code.
Revise the docs about xzkern and add xzkern_with_size. The latter was
added to scripts/Makefile.lib in the commit 7ce7e984ab2b218d ("kbuild:
rename cmd_{bzip2,lzma,lzo,lz4,xzkern,zstd22}").
Omit contact info as MAINTAINERS has it.
Omit other info that is outdated or not relevant in the kernel context.
Co-developed-by: Jia Tan <[email protected]>
Signed-off-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
Documentation/staging/xz.rst | 130 +++++++++++++----------------------
1 file changed, 46 insertions(+), 84 deletions(-)
diff --git a/Documentation/staging/xz.rst b/Documentation/staging/xz.rst
index b2f5ff12a161..301b1eeff411 100644
--- a/Documentation/staging/xz.rst
+++ b/Documentation/staging/xz.rst
@@ -1,3 +1,5 @@
+.. SPDX-License-Identifier: 0BSD
+
============================
XZ data compression in Linux
============================
@@ -6,21 +8,17 @@ Introduction
============
XZ is a general purpose data compression format with high compression
-ratio and relatively fast decompression. The primary compression
-algorithm (filter) is LZMA2. Additional filters can be used to improve
-compression ratio even further. E.g. Branch/Call/Jump (BCJ) filters
-improve compression ratio of executable data.
-
-The XZ decompressor in Linux is called XZ Embedded. It supports
-the LZMA2 filter and optionally also BCJ filters. CRC32 is supported
-for integrity checking. The home page of XZ Embedded is at
-<https://tukaani.org/xz/embedded.html>, where you can find the
-latest version and also information about using the code outside
-the Linux kernel.
+ratio. The XZ decompressor in Linux is called XZ Embedded. It supports
+the LZMA2 filter and optionally also Branch/Call/Jump (BCJ) filters
+for executable code. CRC32 is supported for integrity checking. The
+home page of XZ Embedded is at <https://xz.tukaani.org/xz-embedded/>.
+There you can find the latest version which includes a few optional
+extra features that aren't required in the Linux kernel and information
+about using the code outside the Linux kernel.
For userspace, XZ Utils provide a zlib-like compression library
and a gzip-like command line tool. XZ Utils can be downloaded from
-<https://tukaani.org/xz/>.
+<https://xz.tukaani.org/xz-utils/>.
XZ related components in the kernel
===================================
@@ -29,29 +27,24 @@ The xz_dec module provides XZ decompressor with single-call (buffer
to buffer) and multi-call (stateful) APIs. The usage of the xz_dec
module is documented in include/linux/xz.h.
-The xz_dec_test module is for testing xz_dec. xz_dec_test is not
-useful unless you are hacking the XZ decompressor. xz_dec_test
-allocates a char device major dynamically to which one can write
-.xz files from userspace. The decompressed output is thrown away.
-Keep an eye on dmesg to see diagnostics printed by xz_dec_test.
-See the xz_dec_test source code for the details.
-
For decompressing the kernel image, initramfs, and initrd, there
is a wrapper function in lib/decompress_unxz.c. Its API is the
same as in other decompress_*.c files, which is defined in
include/linux/decompress/generic.h.
-scripts/xz_wrap.sh is a wrapper for the xz command line tool found
-from XZ Utils. The wrapper sets compression options to values suitable
-for compressing the kernel image.
+For kernel makefiles, three commands are provided for use with
+$(call if_changed). They require the xz tool from XZ Utils.
+
+ - $(call if_changed,xzkern) is for compressing the kernel image.
+ It runs the script scripts/xz_wrap.sh which uses arch-optimized
+ options and a big LZMA2 dictionary.
-For kernel makefiles, two commands are provided for use with
-$(call if_needed). The kernel image should be compressed with
-$(call if_needed,xzkern) which will use a BCJ filter and a big LZMA2
-dictionary. It will also append a four-byte trailer containing the
-uncompressed size of the file, which is needed by the boot code.
-Other things should be compressed with $(call if_needed,xzmisc)
-which will use no BCJ filter and 1 MiB LZMA2 dictionary.
+ - $(call if_changed,xzkern_with_size) is like xzkern above but this
+ also appends a four-byte trailer containing the uncompressed size
+ of the file, which is needed by the boot code on some archs.
+
+ - Other things can be compressed with $(call if_needed,xzmisc)
+ which will use no BCJ filter and 1 MiB LZMA2 dictionary.
Notes on compression options
============================
@@ -61,7 +54,7 @@ CRC32, make sure that you don't use some other integrity check type
when encoding files that are supposed to be decoded by the kernel. With
liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32
when encoding. With the xz command line tool, use --check=none or
---check=crc32.
+--check=crc32 to override the default --check=crc64.
Using CRC32 is strongly recommended unless there is some other layer
which will verify the integrity of the uncompressed data anyway.
@@ -71,57 +64,26 @@ by the decoder; you can only change the integrity check type (or
disable it) for the actual uncompressed data.
In userspace, LZMA2 is typically used with dictionary sizes of several
-megabytes. The decoder needs to have the dictionary in RAM, thus big
-dictionaries cannot be used for files that are intended to be decoded
-by the kernel. 1 MiB is probably the maximum reasonable dictionary
-size for in-kernel use (maybe more is OK for initramfs). The presets
-in XZ Utils may not be optimal when creating files for the kernel,
-so don't hesitate to use custom settings. Example::
-
- xz --check=crc32 --lzma2=dict=512KiB inputfile
-
-An exception to above dictionary size limitation is when the decoder
-is used in single-call mode. Decompressing the kernel itself is an
-example of this situation. In single-call mode, the memory usage
-doesn't depend on the dictionary size, and it is perfectly fine to
-use a big dictionary: for maximum compression, the dictionary should
-be at least as big as the uncompressed data itself.
-
-Future plans
-============
-
-Creating a limited XZ encoder may be considered if people think it is
-useful. LZMA2 is slower to compress than e.g. Deflate or LZO even at
-the fastest settings, so it isn't clear if LZMA2 encoder is wanted
-into the kernel.
-
-Support for limited random-access reading is planned for the
-decompression code. I don't know if it could have any use in the
-kernel, but I know that it would be useful in some embedded projects
-outside the Linux kernel.
-
-Conformance to the .xz file format specification
-================================================
-
-There are a couple of corner cases where things have been simplified
-at expense of detecting errors as early as possible. These should not
-matter in practice all, since they don't cause security issues. But
-it is good to know this if testing the code e.g. with the test files
-from XZ Utils.
-
-Reporting bugs
-==============
-
-Before reporting a bug, please check that it's not fixed already
-at upstream. See <https://tukaani.org/xz/embedded.html> to get the
-latest code.
-
-Report bugs to <[email protected]> or visit #tukaani on
-Freenode and talk to Larhzu. I don't actively read LKML or other
-kernel-related mailing lists, so if there's something I should know,
-you should email to me personally or use IRC.
-
-Don't bother Igor Pavlov with questions about the XZ implementation
-in the kernel or about XZ Utils. While these two implementations
-include essential code that is directly based on Igor Pavlov's code,
-these implementations aren't maintained nor supported by him.
+megabytes. The decoder needs to have the dictionary in RAM:
+
+ - In multi-call mode the dictionary is allocated as part of the
+ decoder state. The reasonable maximum dictionary size for in-kernel
+ use will depend on the target hardware: a few megabytes should be
+ fine for desktop systems while 64 KiB to 1 MiB might be more
+ appropriate on some embedded systems.
+
+ - In single-call mode the output buffer is used as the dictionary
+ buffer. That is, the size of the dictionary doesn't affect the
+ decompressor memory usage at all. Only the base data structures
+ are allocated which take a little less than 30 KiB of memory.
+ For the best compression, the dictionary should be at least
+ as big as the uncompressed data. A notable example of single-call
+ mode is decompressing the kernel itself (except on PowerPC).
+
+The compression presets in XZ Utils may not be optimal when creating
+files for the kernel, so don't hesitate to use custom settings to,
+for example, set the dictionary size. Also, xz may produce a smaller
+file in single-threaded mode so setting that explicitly is recommended.
+Example::
+
+ xz --threads=1 --check=crc32 --lzma2=dict=512KiB inputfile
--
2.44.0
Also omit a duplicated check for XZ_DEC_ARM in xz_private.h.
This filter can be used by Squashfs without modifications to the Squashfs
kernel code (only needs support in userspace Squashfs-tools).
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
Notes:
Compared to the first patch I submitted on 2023-11-08
(see <https://lore.kernel.org/lkml/20231108194448.674cd0ad@kaneli/>),
this has a minor tweak to make the for-loop condition faster.
Squashfs-tools Git repository already has support for creating
file systems that use the ARM64 filter.
lib/xz/Kconfig | 5 +++++
lib/xz/xz_dec_bcj.c | 52 ++++++++++++++++++++++++++++++++++++++++++++-
lib/xz/xz_private.h | 7 ++++--
3 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig
index 6b80453d8f54..1166627a87dc 100644
--- a/lib/xz/Kconfig
+++ b/lib/xz/Kconfig
@@ -30,6 +30,11 @@ config XZ_DEC_ARMTHUMB
default y
select XZ_DEC_BCJ
+config XZ_DEC_ARM64
+ bool "ARM64 BCJ filter decoder" if EXPERT
+ default y
+ select XZ_DEC_BCJ
+
config XZ_DEC_SPARC
bool "SPARC BCJ filter decoder" if EXPERT
default y
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
index e0b4bf4999c0..941198a8a55b 100644
--- a/lib/xz/xz_dec_bcj.c
+++ b/lib/xz/xz_dec_bcj.c
@@ -23,7 +23,8 @@ struct xz_dec_bcj {
BCJ_IA64 = 6, /* Big or little endian */
BCJ_ARM = 7, /* Little endian only */
BCJ_ARMTHUMB = 8, /* Little endian only */
- BCJ_SPARC = 9 /* Big or little endian */
+ BCJ_SPARC = 9, /* Big or little endian */
+ BCJ_ARM64 = 10 /* AArch64 */
} type;
/*
@@ -346,6 +347,47 @@ static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
}
#endif
+#ifdef XZ_DEC_ARM64
+static size_t bcj_arm64(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
+{
+ size_t i;
+ uint32_t instr;
+ uint32_t addr;
+
+ size &= ~(size_t)3;
+
+ for (i = 0; i < size; i += 4) {
+ instr = get_unaligned_le32(buf + i);
+
+ if ((instr >> 26) == 0x25) {
+ /* BL instruction */
+ addr = instr - ((s->pos + (uint32_t)i) >> 2);
+ instr = 0x94000000 | (addr & 0x03FFFFFF);
+ put_unaligned_le32(instr, buf + i);
+
+ } else if ((instr & 0x9F000000) == 0x90000000) {
+ /* ADRP instruction */
+ addr = ((instr >> 29) & 3) | ((instr >> 3) & 0x1FFFFC);
+
+ /* Only convert values in the range +/-512 MiB. */
+ if ((addr + 0x020000) & 0x1C0000)
+ continue;
+
+ addr -= (s->pos + (uint32_t)i) >> 12;
+
+ instr &= 0x9000001F;
+ instr |= (addr & 3) << 29;
+ instr |= (addr & 0x03FFFC) << 3;
+ instr |= (0U - (addr & 0x020000)) & 0xE00000;
+
+ put_unaligned_le32(instr, buf + i);
+ }
+ }
+
+ return i;
+}
+#endif
+
/*
* Apply the selected BCJ filter. Update *pos and s->pos to match the amount
* of data that got filtered.
@@ -392,6 +434,11 @@ static void bcj_apply(struct xz_dec_bcj *s,
case BCJ_SPARC:
filtered = bcj_sparc(s, buf, size);
break;
+#endif
+#ifdef XZ_DEC_ARM64
+ case BCJ_ARM64:
+ filtered = bcj_arm64(s, buf, size);
+ break;
#endif
default:
/* Never reached but silence compiler warnings. */
@@ -565,6 +612,9 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id)
#endif
#ifdef XZ_DEC_SPARC
case BCJ_SPARC:
+#endif
+#ifdef XZ_DEC_ARM64
+ case BCJ_ARM64:
#endif
break;
diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h
index 811add814ae4..307e0de8c260 100644
--- a/lib/xz/xz_private.h
+++ b/lib/xz/xz_private.h
@@ -36,6 +36,9 @@
# ifdef CONFIG_XZ_DEC_SPARC
# define XZ_DEC_SPARC
# endif
+# ifdef CONFIG_XZ_DEC_ARM64
+# define XZ_DEC_ARM64
+# endif
# ifdef CONFIG_XZ_DEC_MICROLZMA
# define XZ_DEC_MICROLZMA
# endif
@@ -97,9 +100,9 @@
*/
#ifndef XZ_DEC_BCJ
# if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \
- || defined(XZ_DEC_IA64) || defined(XZ_DEC_ARM) \
+ || defined(XZ_DEC_IA64) \
|| defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \
- || defined(XZ_DEC_SPARC)
+ || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64)
# define XZ_DEC_BCJ
# endif
#endif
--
2.44.0
The commit faa16bc404d72a5a ("lib: Use existing define with polynomial")
in 2018 added a dependency on <linux/crc32poly.h> to avoid duplicating
the same constant in multiple files. Two months later it was found to be
a bad idea and the commit 242cdad873a75652 ("lib/xz: Put CRC32_POLY_LE
in xz_private.h") added the definition of CRC32_POLY_LE macro into
xz_private.h to avoid including <linux/crc32poly.h>.
xz_private.h is a wrong place for it too. Revert back to the upstream
version which has the poly in xz_crc32_init() in xz_crc32.c.
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
lib/xz/xz_crc32.c | 2 +-
lib/xz/xz_private.h | 4 ----
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c
index 30b8a27110b1..effdf34ec48d 100644
--- a/lib/xz/xz_crc32.c
+++ b/lib/xz/xz_crc32.c
@@ -28,7 +28,7 @@ STATIC_RW_DATA uint32_t xz_crc32_table[256];
XZ_EXTERN void xz_crc32_init(void)
{
- const uint32_t poly = CRC32_POLY_LE;
+ const uint32_t poly = 0xEDB88320;
uint32_t i;
uint32_t j;
diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h
index 2412a5d54801..811add814ae4 100644
--- a/lib/xz/xz_private.h
+++ b/lib/xz/xz_private.h
@@ -104,10 +104,6 @@
# endif
#endif
-#ifndef CRC32_POLY_LE
-#define CRC32_POLY_LE 0xedb88320
-#endif
-
/*
* Allocate memory for LZMA2 decoder. xz_dec_lzma2_reset() must be used
* before calling xz_dec_lzma2_run().
--
2.44.0
Use LZMA2 options that match the arch-specific alignment of instructions.
This change reduces compressed kernel size 0-2 % depending on the arch.
On 1-byte-aligned x86 it makes no difference and on 4-byte-aligned archs
it helps the most.
Use the ARM-Thumb filter for ARM-Thumb2 kernels. This reduces compressed
kernel size about 5 %.[1] Previously such kernels were compressed using
the ARM filter which didn't do anything useful with ARM-Thumb2 code.
Add BCJ filter support for ARM64 and RISC-V. On ARM64 the compressed
kernel size is reduced about 5 % and on RISC-V by 7-8 % compared to
unfiltered XZ or plain LZMA. However:
- arch/arm64/boot/Makefile and arch/riscv/boot/Makefile don't include
the build rule (two lines) for XZ support even though they support
six other compressors. It would be trivial to add the rule but boot
loaders would need XZ support too.
- A new enough version of the xz tool is required: 5.4.0 for ARM64 and
5.6.0 for RISC-V. With an old xz version a message is printed to
standard error and the kernel is compressed without the filter.
Update lib/decompress_unxz.c to match the changes to xz_wrap.sh.
Update the CONFIG_KERNEL_XZ help text in init/Kconfig:
- Add the RISC-V and ARM64 filters.
- Clarify that the PowerPC filter is for big endian only.
- Omit IA-64.
Link: https://lore.kernel.org/lkml/[email protected]/ [1]
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
init/Kconfig | 5 +-
lib/decompress_unxz.c | 14 ++++-
scripts/xz_wrap.sh | 141 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 151 insertions(+), 9 deletions(-)
diff --git a/init/Kconfig b/init/Kconfig
index f3ea5dea9c85..785e15aa5395 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -308,8 +308,9 @@ config KERNEL_XZ
BCJ filters which can improve compression ratio of executable
code. The size of the kernel is about 30% smaller with XZ in
comparison to gzip. On architectures for which there is a BCJ
- filter (i386, x86_64, ARM, IA-64, PowerPC, and SPARC), XZ
- will create a few percent smaller kernel than plain LZMA.
+ filter (i386, x86_64, ARM, ARM64, RISC-V, big endian PowerPC,
+ and SPARC), XZ will create a few percent smaller kernel than
+ plain LZMA.
The speed is about the same as with LZMA: The decompression
speed of XZ is better than that of bzip2 but worse than gzip
diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
index 46aa3be13fc5..cae00395d7a6 100644
--- a/lib/decompress_unxz.c
+++ b/lib/decompress_unxz.c
@@ -126,11 +126,21 @@
#ifdef CONFIG_X86
# define XZ_DEC_X86
#endif
-#ifdef CONFIG_PPC
+#if defined(CONFIG_PPC) && defined(CONFIG_CPU_BIG_ENDIAN)
# define XZ_DEC_POWERPC
#endif
#ifdef CONFIG_ARM
-# define XZ_DEC_ARM
+# ifdef CONFIG_THUMB2_KERNEL
+# define XZ_DEC_ARMTHUMB
+# else
+# define XZ_DEC_ARM
+# endif
+#endif
+#ifdef CONFIG_ARM64
+# define XZ_DEC_ARM64
+#endif
+#ifdef CONFIG_RISCV
+# define XZ_DEC_RISCV
#endif
#ifdef CONFIG_SPARC
# define XZ_DEC_SPARC
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index c8c36441ab70..5bdf0c35cc85 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -6,14 +6,145 @@
#
# Author: Lasse Collin <[email protected]>
+# This has specialized settings for the following archs. However,
+# XZ-compressed kernel isn't currently supported on every listed arch.
+#
+# Arch Align Notes
+# arm 2/4 ARM and ARM-Thumb2
+# arm64 4
+# csky 2
+# loongarch 4
+# mips 2/4 MicroMIPS is 2-byte aligned
+# parisc 4
+# powerpc 4 Uses its own wrapper for compressors instead of this.
+# riscv 2/4
+# s390 2
+# sh 2
+# sparc 4
+# x86 1
+
+# A few archs use 2-byte or 4-byte aligned instructions depending on
+# the kernel config. This function is used to check if the relevant
+# config option is set to "y".
+is_enabled()
+{
+ grep -q "^$1=y$" include/config/auto.conf
+}
+
+# Set XZ_VERSION (and LIBLZMA_VERSION). This is needed to disable features
+# that aren't available in old XZ Utils versions.
+eval "$($XZ --robot --version)" || exit
+
+# Assume that no BCJ filter is available.
BCJ=
-LZMA2OPTS=
+# Set the instruction alignment to 1, 2, or 4 bytes.
+#
+# Set the BCJ filter if one is available.
+# It must match the #ifdef usage in lib/decompress_unxz.c.
case $SRCARCH in
- x86) BCJ=--x86 ;;
- powerpc) BCJ=--powerpc ;;
- arm) BCJ=--arm ;;
- sparc) BCJ=--sparc ;;
+ arm)
+ if is_enabled CONFIG_THUMB2_KERNEL; then
+ ALIGN=2
+ BCJ=--armthumb
+ else
+ ALIGN=4
+ BCJ=--arm
+ fi
+ ;;
+
+ arm64)
+ ALIGN=4
+
+ # ARM64 filter was added in XZ Utils 5.4.0.
+ if [ "$XZ_VERSION" -ge 50040002 ]; then
+ BCJ=--arm64
+ else
+ echo "$0: Upgrading to xz >= 5.4.0" \
+ "would enable the ARM64 filter" \
+ "for better compression" >&2
+ fi
+ ;;
+
+ csky)
+ ALIGN=2
+ ;;
+
+ loongarch)
+ ALIGN=4
+ ;;
+
+ mips)
+ if is_enabled CONFIG_CPU_MICROMIPS; then
+ ALIGN=2
+ else
+ ALIGN=4
+ fi
+ ;;
+
+ parisc)
+ ALIGN=4
+ ;;
+
+ powerpc)
+ ALIGN=4
+
+ # The filter is only for big endian instruction encoding.
+ if is_enabled CONFIG_CPU_BIG_ENDIAN; then
+ BCJ=--powerpc
+ fi
+ ;;
+
+ riscv)
+ if is_enabled CONFIG_RISCV_ISA_C; then
+ ALIGN=2
+ else
+ ALIGN=4
+ fi
+
+ # RISC-V filter was added in XZ Utils 5.6.0.
+ if [ "$XZ_VERSION" -ge 50060002 ]; then
+ BCJ=--riscv
+ else
+ echo "$0: Upgrading to xz >= 5.6.0" \
+ "would enable the RISC-V filter" \
+ "for better compression" >&2
+ fi
+ ;;
+
+ s390)
+ ALIGN=2
+ ;;
+
+ sh)
+ ALIGN=2
+ ;;
+
+ sparc)
+ ALIGN=4
+ BCJ=--sparc
+ ;;
+
+ x86)
+ ALIGN=1
+ BCJ=--x86
+ ;;
+
+ *)
+ echo "$0: Arch-specific tuning is missing for '$SRCARCH'" >&2
+
+ # Guess 2-byte-aligned instructions. Guessing too low
+ # should hurt less than guessing too high.
+ ALIGN=2
+ ;;
+esac
+
+# Select the LZMA2 options matching the instruction alignment.
+case $ALIGN in
+ 1) LZMA2OPTS= ;;
+ 2) LZMA2OPTS=lp=1 ;;
+ 4) LZMA2OPTS=lp=2,lc=2 ;;
+ *) echo "$0: ALIGN wrong or missing" >&2; exit 1 ;;
esac
# Use single-threaded mode because it compresses a little better
--
2.44.0
This only affects kernel image compression, not any other xz usage.
Desktop kernels on x86-64 are already around 60 MiB. Using a dictionary
larger than 32 MiB should have no downsides nowadays as anyone building
the kernel should have plenty of RAM. 128 MiB dictionary needs 1346 MiB
of RAM with xz versions 5.0.x - 5.6.x in single-threaded mode. On archs
that use xz_wrap.sh, kernel decompression is done in single-call mode so
a larger dictionary doesn't affect boot-time memory requirements.
xz >= 5.6.0 uses multithreaded mode by default which compresses slightly
worse than single-threaded mode. Kernel compression rarely used more
than one thread anyway because with 32 MiB dictionary size the default
block size was 96 MiB in multithreaded mode. So only a single thread
was used anyway unless the kernel was over 96 MiB.
Comparison to CONFIG_KERNEL_LZMA: It uses "lzma -9" which mapped to
32 MiB dictionary in LZMA Utils 4.32.7 (the final release in 2008).
Nowadays the lzma tool on most systems is from XZ Utils where -9 maps
to 64 MiB dictionary. So using a 32 MiB dictionary with CONFIG_KERNEL_XZ
may have compressed big kernels slightly worse than the old LZMA option.
Comparison to CONFIG_KERNEL_ZSTD: zstd uses 128 MiB dictionary.
Reviewed-by: Jia Tan <[email protected]>
Signed-off-by: Lasse Collin <[email protected]>
---
scripts/xz_wrap.sh | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index bb760b721b2c..c8c36441ab70 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -16,4 +16,15 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+# Use single-threaded mode because it compresses a little better
+# (and uses less RAM) than multithreaded mode.
+#
+# For the best compression, the dictionary size shouldn't be
+# smaller than the uncompressed kernel. 128 MiB dictionary
+# needs less than 1400 MiB of RAM in single-threaded mode.
+#
+# On the archs that use this script to compress the kernel,
+# decompression in the preboot code is done in single-call mode.
+# Thus the dictionary size doesn't affect the memory requirements
+# of the preboot decompressor at all.
+exec $XZ --check=crc32 --threads=1 $BCJ --lzma2=$LZMA2OPTS,dict=128MiB
--
2.44.0
On Wed, Mar 20, 2024 at 08:38:33PM +0200, Lasse Collin wrote:
> XZ Embedded, the upstream project, switched from public domain to the
> BSD Zero Clause License (0BSD). Now matching SPDX license identifiers
> can be added.
>
> The new ARM64 and RISC-V filters can be used by Squashfs.
>
> Account for the default threading change made in the xz command line
> tool version 5.6.0. Tweak kernel compression options for archs that
> support XZ compressed kernel.
>
> Documentation was revised. There are minor cleanups too.
>
> Lasse Collin (11):
> MAINTAINERS: Add XZ Embedded maintainers
> LICENSES: Add 0BSD license text
> xz: Switch from public domain to BSD Zero Clause License (0BSD)
> xz: Documentation/staging/xz.rst: Revise thoroughly
> xz: Fix comments and coding style
> xz: Cleanup CRC32 edits from 2018
> xz: Optimize for-loop conditions in the BCJ decoders
> xz: Add ARM64 BCJ filter
> xz: Add RISC-V BCJ filter
> xz: Use 128 MiB dictionary and force single-threaded mode
> xz: Adjust arch-specific options for better kernel compression
>
> Documentation/staging/xz.rst | 130 ++++++++---------------
> LICENSES/deprecated/0BSD | 23 ++++
> MAINTAINERS | 14 +++
> include/linux/decompress/unxz.h | 5 +-
> include/linux/xz.h | 5 +-
> init/Kconfig | 5 +-
> lib/decompress_unxz.c | 39 ++++---
> lib/xz/Kconfig | 13 ++-
> lib/xz/xz_crc32.c | 7 +-
> lib/xz/xz_dec_bcj.c | 183 ++++++++++++++++++++++++++++++--
> lib/xz/xz_dec_lzma2.c | 5 +-
> lib/xz/xz_dec_stream.c | 5 +-
> lib/xz/xz_dec_syms.c | 16 +--
> lib/xz/xz_dec_test.c | 12 +--
> lib/xz/xz_lzma2.h | 5 +-
> lib/xz/xz_private.h | 20 ++--
> lib/xz/xz_stream.h | 7 +-
> scripts/Makefile.lib | 13 ++-
> scripts/xz_wrap.sh | 157 +++++++++++++++++++++++++--
> 19 files changed, 487 insertions(+), 177 deletions(-)
> create mode 100644 LICENSES/deprecated/0BSD
Andrew (and anyone else), please do not take this code right now.
Until the backdooring of upstream xz[1] is fully understood, we should not
accept any code from Jia Tan, Lasse Collin, or any other folks associated
with tukaani.org. It appears the domain, or at least credentials
associated with Jia Tan, have been used to create an obfuscated ssh
server backdoor via the xz upstream releases since at least 5.6.0.
Without extensive analysis, we should not take any associated code.
It may be worth doing some retrospective analysis of past contributions
as well...
Lasse, are you able to comment about what is going on here?
-Kees
[1] https://www.openwall.com/lists/oss-security/2024/03/29/4
--
Kees Cook
"Kees Cook" <[email protected]> writes:
> On Wed, Mar 20, 2024 at 08:38:33PM +0200, Lasse Collin wrote:
>> XZ Embedded, the upstream project, switched from public domain to the
>> BSD Zero Clause License (0BSD). Now matching SPDX license identifiers
>> can be added.
>>
>> The new ARM64 and RISC-V filters can be used by Squashfs.
>>
>> Account for the default threading change made in the xz command line
>> tool version 5.6.0. Tweak kernel compression options for archs that
>> support XZ compressed kernel.
>>
>> Documentation was revised. There are minor cleanups too.
>>
>> Lasse Collin (11):
>> MAINTAINERS: Add XZ Embedded maintainers
>> LICENSES: Add 0BSD license text
>> xz: Switch from public domain to BSD Zero Clause License (0BSD)
>> xz: Documentation/staging/xz.rst: Revise thoroughly
>> xz: Fix comments and coding style
>> xz: Cleanup CRC32 edits from 2018
>> xz: Optimize for-loop conditions in the BCJ decoders
>> xz: Add ARM64 BCJ filter
>> xz: Add RISC-V BCJ filter
>> xz: Use 128 MiB dictionary and force single-threaded mode
>> xz: Adjust arch-specific options for better kernel compression
>>
>> Documentation/staging/xz.rst | 130 ++++++++---------------
>> LICENSES/deprecated/0BSD | 23 ++++
>> MAINTAINERS | 14 +++
>> include/linux/decompress/unxz.h | 5 +-
>> include/linux/xz.h | 5 +-
>> init/Kconfig | 5 +-
>> lib/decompress_unxz.c | 39 ++++---
>> lib/xz/Kconfig | 13 ++-
>> lib/xz/xz_crc32.c | 7 +-
>> lib/xz/xz_dec_bcj.c | 183 ++++++++++++++++++++++++++++++--
>> lib/xz/xz_dec_lzma2.c | 5 +-
>> lib/xz/xz_dec_stream.c | 5 +-
>> lib/xz/xz_dec_syms.c | 16 +--
>> lib/xz/xz_dec_test.c | 12 +--
>> lib/xz/xz_lzma2.h | 5 +-
>> lib/xz/xz_private.h | 20 ++--
>> lib/xz/xz_stream.h | 7 +-
>> scripts/Makefile.lib | 13 ++-
>> scripts/xz_wrap.sh | 157 +++++++++++++++++++++++++--
>> 19 files changed, 487 insertions(+), 177 deletions(-)
>> create mode 100644 LICENSES/deprecated/0BSD
>
> Andrew (and anyone else), please do not take this code right now.
>
> Until the backdooring of upstream xz[1] is fully understood, we should not
> accept any code from Jia Tan, Lasse Collin, or any other folks associated
> with tukaani.org. It appears the domain, or at least credentials
> associated with Jia Tan, have been used to create an obfuscated ssh
> server backdoor via the xz upstream releases since at least 5.6.0.
> Without extensive analysis, we should not take any associated code.
> It may be worth doing some retrospective analysis of past contributions
> as well...
>
> Lasse, are you able to comment about what is going on here?
FWIW, it looks like this series has been in linux-next for a few days.
Maybe it needs to come out, for now at least?
jon
On Fri, Mar 29, 2024 at 02:51:41PM -0600, Jonathan Corbet wrote:
> "Kees Cook" <[email protected]> writes:
>
> > On Wed, Mar 20, 2024 at 08:38:33PM +0200, Lasse Collin wrote:
> >> XZ Embedded, the upstream project, switched from public domain to the
> >> BSD Zero Clause License (0BSD). Now matching SPDX license identifiers
> >> can be added.
> >>
> >> The new ARM64 and RISC-V filters can be used by Squashfs.
> >>
> >> Account for the default threading change made in the xz command line
> >> tool version 5.6.0. Tweak kernel compression options for archs that
> >> support XZ compressed kernel.
> >>
> >> Documentation was revised. There are minor cleanups too.
> >>
> >> Lasse Collin (11):
> >> MAINTAINERS: Add XZ Embedded maintainers
> >> LICENSES: Add 0BSD license text
> >> xz: Switch from public domain to BSD Zero Clause License (0BSD)
> >> xz: Documentation/staging/xz.rst: Revise thoroughly
> >> xz: Fix comments and coding style
> >> xz: Cleanup CRC32 edits from 2018
> >> xz: Optimize for-loop conditions in the BCJ decoders
> >> xz: Add ARM64 BCJ filter
> >> xz: Add RISC-V BCJ filter
> >> xz: Use 128 MiB dictionary and force single-threaded mode
> >> xz: Adjust arch-specific options for better kernel compression
> >>
> >> Documentation/staging/xz.rst | 130 ++++++++---------------
> >> LICENSES/deprecated/0BSD | 23 ++++
> >> MAINTAINERS | 14 +++
> >> include/linux/decompress/unxz.h | 5 +-
> >> include/linux/xz.h | 5 +-
> >> init/Kconfig | 5 +-
> >> lib/decompress_unxz.c | 39 ++++---
> >> lib/xz/Kconfig | 13 ++-
> >> lib/xz/xz_crc32.c | 7 +-
> >> lib/xz/xz_dec_bcj.c | 183 ++++++++++++++++++++++++++++++--
> >> lib/xz/xz_dec_lzma2.c | 5 +-
> >> lib/xz/xz_dec_stream.c | 5 +-
> >> lib/xz/xz_dec_syms.c | 16 +--
> >> lib/xz/xz_dec_test.c | 12 +--
> >> lib/xz/xz_lzma2.h | 5 +-
> >> lib/xz/xz_private.h | 20 ++--
> >> lib/xz/xz_stream.h | 7 +-
> >> scripts/Makefile.lib | 13 ++-
> >> scripts/xz_wrap.sh | 157 +++++++++++++++++++++++++--
> >> 19 files changed, 487 insertions(+), 177 deletions(-)
> >> create mode 100644 LICENSES/deprecated/0BSD
> >
> > Andrew (and anyone else), please do not take this code right now.
> >
> > Until the backdooring of upstream xz[1] is fully understood, we should not
> > accept any code from Jia Tan, Lasse Collin, or any other folks associated
> > with tukaani.org. It appears the domain, or at least credentials
> > associated with Jia Tan, have been used to create an obfuscated ssh
> > server backdoor via the xz upstream releases since at least 5.6.0.
> > Without extensive analysis, we should not take any associated code.
> > It may be worth doing some retrospective analysis of past contributions
> > as well...
> >
> > Lasse, are you able to comment about what is going on here?
>
> FWIW, it looks like this series has been in linux-next for a few days.
> Maybe it needs to come out, for now at least?
Yes, for sure. Andrew, just so it's explicitly clear: please remove this
series from -mm for now, until the situation is better understood.
Thanks!
-Kees
--
Kees Cook
On Fri, 29 Mar 2024 14:51:41 -0600 Jonathan Corbet <[email protected]> wrote:
> > Andrew (and anyone else), please do not take this code right now.
> >
> > Until the backdooring of upstream xz[1] is fully understood, we should not
> > accept any code from Jia Tan, Lasse Collin, or any other folks associated
> > with tukaani.org. It appears the domain, or at least credentials
> > associated with Jia Tan, have been used to create an obfuscated ssh
> > server backdoor via the xz upstream releases since at least 5.6.0.
> > Without extensive analysis, we should not take any associated code.
> > It may be worth doing some retrospective analysis of past contributions
> > as well...
> >
> > Lasse, are you able to comment about what is going on here?
>
> FWIW, it looks like this series has been in linux-next for a few days.
> Maybe it needs to come out, for now at least?
Yes, I have removed that series.
Under the light of the recent xz backdoor, I should note that this
patch (patch 11) does:
> +# Set XZ_VERSION (and LIBLZMA_VERSION). This is needed to disable features
> +# that aren't available in old XZ Utils versions.
> +eval "$($XZ --robot --version)" || exit
> +
in order to do
> + arm64)
> + ALIGN=4
> +
> + # ARM64 filter was added in XZ Utils 5.4.0.
> + if [ "$XZ_VERSION" -ge 50040002 ]; then
> + BCJ=--arm64
> + else
> + echo "$0: Upgrading to xz >= 5.4.0" \
> + "would enable the ARM64 filter" \
> + "for better compression" >&2
> + fi
> + ;;
and
> + # RISC-V filter was added in XZ Utils 5.6.0.
> + if [ "$XZ_VERSION" -ge 50060002 ]; then
> + BCJ=--riscv
> + else
> + echo "$0: Upgrading to xz >= 5.6.0" \
> + "would enable the RISC-V filter" \
> + "for better compression" >&2
> + fi
>
which was noted on Hacker News as a potential gadget of
exploitation[1]. Thanks Vegard for bringing it up[2].
A compromised $XZ could modify the build files directly in C, or even
produce a file that decompresses into a kernel with added evil
instructions, at a quite near level to Reflections on Trusting Trust.
Nonetheless, execution of high level shell script would probably be
more useful for an attacker that has to surreptitiously include their
backdoor, as it would only require a few bytes (e.g. a sed call) when
compared to coding that in C.
So, in the spirit of keeping a fair amount of paranoia, and since it
doesn't do any harm, any such code should be failproofed to ensure it
can only import the expected shell variables with the right format[3]:
eval "$($XZ --robot --version | grep '^\(XZ\|LIBLZMA\)_VERSION=[0-9]*$')" || exit
Regards
[1] https://news.ycombinator.com/item?id=39869715
[2] https://www.openwall.com/lists/oss-security/2024/03/30/11
[3] Actually, LIBLZMA_VERSION isn't used, only XZ_VERSION. Being
generous and accepting that one as well. :)
On 2024-03-29 Andrew Morton wrote:
> On Fri, 29 Mar 2024 14:51:41 -0600 Jonathan Corbet <[email protected]>
> wrote:
>
> > > Andrew (and anyone else), please do not take this code right now.
> > >
> > > Until the backdooring of upstream xz[1] is fully understood, we
> > > should not accept any code from Jia Tan, Lasse Collin, or any
> > > other folks associated with tukaani.org. It appears the domain,
> > > or at least credentials associated with Jia Tan, have been used
> > > to create an obfuscated ssh server backdoor via the xz upstream
> > > releases since at least 5.6.0. Without extensive analysis, we
> > > should not take any associated code. It may be worth doing some
> > > retrospective analysis of past contributions as well...
> > >
> > > Lasse, are you able to comment about what is going on here?
> >
> > FWIW, it looks like this series has been in linux-next for a few
> > days. Maybe it needs to come out, for now at least?
>
> Yes, I have removed that series.
Thank you. None of these patches are urgent. I'm on a holiday and only
happened to look at my emails and it seems to be a major mess.
My proper investigation efforts likely start in the first days of
April. That is, I currently know only a few facts which alone are bad
enough.
Info will be updated here: https://tukaani.org/xz-backdoor/
--
Lasse Collin
Just in case it hasn't been brought to the kernel's attention, the xz
project has been compromised in its upstream repo, and versions 5.6.0
and 5.6.1 attempt to load a backdoor into SSH. I suggest any patches
associated with Lasse Collin, Jia Tan, or tukaani.org be held until
that matter is fully resolved. And all their previous work needs to be
re-examined with this in mind.
--Jonathan Bennett
On 3/20/24 1:38 PM, Lasse Collin wrote:
> XZ Embedded, the upstream project, switched from public domain to the
> BSD Zero Clause License (0BSD). Now matching SPDX license identifiers
> can be added.
>
> The new ARM64 and RISC-V filters can be used by Squashfs.
>
> Account for the default threading change made in the xz command line
> tool version 5.6.0. Tweak kernel compression options for archs that
> support XZ compressed kernel.
>
> Documentation was revised. There are minor cleanups too.
>
> Lasse Collin (11):
> MAINTAINERS: Add XZ Embedded maintainers
> LICENSES: Add 0BSD license text
> xz: Switch from public domain to BSD Zero Clause License (0BSD)
> xz: Documentation/staging/xz.rst: Revise thoroughly
> xz: Fix comments and coding style
> xz: Cleanup CRC32 edits from 2018
> xz: Optimize for-loop conditions in the BCJ decoders
> xz: Add ARM64 BCJ filter
> xz: Add RISC-V BCJ filter
> xz: Use 128 MiB dictionary and force single-threaded mode
> xz: Adjust arch-specific options for better kernel compression
>
> Documentation/staging/xz.rst | 130 ++++++++---------------
> LICENSES/deprecated/0BSD | 23 ++++
> MAINTAINERS | 14 +++
> include/linux/decompress/unxz.h | 5 +-
> include/linux/xz.h | 5 +-
> init/Kconfig | 5 +-
> lib/decompress_unxz.c | 39 ++++---
> lib/xz/Kconfig | 13 ++-
> lib/xz/xz_crc32.c | 7 +-
> lib/xz/xz_dec_bcj.c | 183 ++++++++++++++++++++++++++++++--
> lib/xz/xz_dec_lzma2.c | 5 +-
> lib/xz/xz_dec_stream.c | 5 +-
> lib/xz/xz_dec_syms.c | 16 +--
> lib/xz/xz_dec_test.c | 12 +--
> lib/xz/xz_lzma2.h | 5 +-
> lib/xz/xz_private.h | 20 ++--
> lib/xz/xz_stream.h | 7 +-
> scripts/Makefile.lib | 13 ++-
> scripts/xz_wrap.sh | 157 +++++++++++++++++++++++++--
> 19 files changed, 487 insertions(+), 177 deletions(-)
> create mode 100644 LICENSES/deprecated/0BSD
>
On 2024-03-31 [email protected] wrote:
> Under the light of the recent xz backdoor, I should note that this
> patch (patch 11) does:
>
> > +# Set XZ_VERSION (and LIBLZMA_VERSION). This is needed to disable
> > features +# that aren't available in old XZ Utils versions.
> > +eval "$($XZ --robot --version)" || exit
The eval method has been on the xz man page for a very long time but I
agree that due to the recent events the above method is not ideal.
It can break also if XZ_OPT or XZ_DEFAULTS contains something that they
usually shouldn't. For example, XZ_OPT=--help would make the above eval
method run the output of $XZ --help.
> So, in the spirit of keeping a fair amount of paranoia, and since it
> doesn't do any harm, any such code should be failproofed to ensure it
> can only import the expected shell variables with the right format[3]:
>
> eval "$($XZ --robot --version | grep '^\(XZ\|LIBLZMA\)_VERSION=[0-9]*$')" || exit
I would rather get rid of eval. I committed the following to the
upstream repository:
XZ_VERSION=$($XZ --robot --version | sed -n 's/^XZ_VERSION=//p') || exit
Thanks!
--
Lasse Collin
On 2024-04-03 Lasse Collin wrote:
> On 2024-03-31 [email protected] wrote:
> > So, in the spirit of keeping a fair amount of paranoia, and since it
> > doesn't do any harm, any such code should be failproofed to ensure
> > it can only import the expected shell variables with the right
> > format[3]:
> >
> > eval "$($XZ --robot --version | grep
> > '^\(XZ\|LIBLZMA\)_VERSION=[0-9]*$')" || exit
>
> I would rather get rid of eval. I committed the following to the
> upstream repository:
>
> XZ_VERSION=$($XZ --robot --version | sed -n 's/^XZ_VERSION=//p') ||
> exit
Both my new version and the suggested eval+grep version have error
detection issues:
- With the eval+grep version, if there are no matches, eval gets an
empty string as an argument in which case eval's exit status is
zero and "exit" won't be run. Exit status from $XZ is ignored.
XZ_VERSION won't be set or it might be inherited from the
environment.
- With $XZ ... | sed ..., the exit status of $XZ is ignored. sed
will exit with 0 and thus "exit" won't be run even if $XZ fails.
Upstream I changed to this:
XZ_VERSION=$($XZ --robot --version) || exit
XZ_VERSION=$(printf '%s\n' "$XZ_VERSION" | sed -n 's/^XZ_VERSION=//p')
If output from $XZ is weird, XZ_VERSION might still become weird too.
But the way the variable is used later should at worst result in
"integer expression expected" error message.
I think the above is a good enough balance for a shell script like
this.
--
Lasse Collin