This is the second part of the bzip2/lzma patch
The bzip patch is based on an idea by Christian Ludwig, includes support for
compressing the kernel with bzip2 or lzma rather than gzip. Both
compressors give smaller sizes than gzip. Lzma's decompresses faster
than bzip2.
It also supports ramdisks and initramfs' compressed using these two
compressors.
The functionality has been successfully used for a couple of years by
the udpcast project
This version applies to "tip" kernel 2.6.28
This part contains:
- support for new compressions (bzip2 and lzma) in initramfs and
old-style ramdisk
- config dialog for kernel compression (but new kernel compressions
not yet supported)
Signed-off-by: Alain Knaff <[email protected]>
---
diff -urNp library/drivers/block/Kconfig generic/drivers/block/Kconfig
--- library/drivers/block/Kconfig 2009-01-04 11:56:38.000000000 +0100
+++ generic/drivers/block/Kconfig 2009-01-04 21:57:13.000000000 +0100
@@ -358,6 +358,31 @@ config BLK_DEV_XIP
will prevent RAM block device backing store memory from being
allocated from highmem (only a problem for highmem systems).
+config RD_BZIP2
+ bool "Initial ramdisk compressed using bzip2"
+ default n
+ depends on BLK_DEV_INITRD=y
+ help
+ Support loading of a bzip2 encoded initial ramdisk or cpio buffer
+ If unsure, say N.
+
+config RD_LZMA
+ bool "Initial ramdisk compressed using lzma"
+ default n
+ depends on BLK_DEV_INITRD=y
+ help
+ Support loading of a lzma encoded initial ramdisk or cpio buffer
+ If unsure, say N.
+
+config RD_GZIP
+ bool "Initial ramdisk compressed using gzip"
+ default y
+ depends on BLK_DEV_INITRD=y
+ select ZLIB_INFLATE
+ help
+ Support loading of a gzip encoded initial ramdisk or cpio buffer.
+ If unsure, say Y.
+
config CDROM_PKTCDVD
tristate "Packet writing on CD/DVD media"
depends on !UML
diff -urNp library/init/do_mounts_rd.c generic/init/do_mounts_rd.c
--- library/init/do_mounts_rd.c 2009-01-04 11:56:42.000000000 +0100
+++ generic/init/do_mounts_rd.c 2009-01-04 21:57:13.000000000 +0100
@@ -10,6 +10,12 @@
#include "do_mounts.h"
+#include <linux/decompress/generic.h>
+
+#include <linux/decompress/bunzip2.h>
+#include <linux/decompress/unlzma.h>
+#include <linux/decompress/inflate.h>
+
int __initdata rd_prompt = 1;/* 1 = prompt for RAM disk, 0 = don't prompt */
static int __init prompt_ramdisk(char *str)
@@ -28,7 +34,7 @@ static int __init ramdisk_start_setup(ch
}
__setup("ramdisk_start=", ramdisk_start_setup);
-static int __init crd_load(int in_fd, int out_fd);
+static int __init crd_load(int in_fd, int out_fd, decompress_fn deco);
/*
* This routine tries to find a RAM disk image to load, and returns the
@@ -44,7 +50,7 @@ static int __init crd_load(int in_fd, in
* gzip
*/
static int __init
-identify_ramdisk_image(int fd, int start_block)
+identify_ramdisk_image(int fd, int start_block, decompress_fn *decompressor)
{
const int size = 512;
struct minix_super_block *minixsb;
@@ -70,6 +76,7 @@ identify_ramdisk_image(int fd, int start
sys_lseek(fd, start_block * BLOCK_SIZE, 0);
sys_read(fd, buf, size);
+#ifdef CONFIG_RD_GZIP
/*
* If it matches the gzip magic numbers, return 0
*/
@@ -77,9 +84,39 @@ identify_ramdisk_image(int fd, int start
printk(KERN_NOTICE
"RAMDISK: Compressed image found at block %d\n",
start_block);
+ *decompressor = gunzip;
+ nblocks = 0;
+ goto done;
+ }
+#endif
+
+#ifdef CONFIG_RD_BZIP2
+ /*
+ * If it matches the bzip2 magic numbers, return -1
+ */
+ if (buf[0] == 0x42 && (buf[1] == 0x5a)) {
+ printk(KERN_NOTICE
+ "RAMDISK: Bzipped image found at block %d\n",
+ start_block);
+ *decompressor = bunzip2;
nblocks = 0;
goto done;
}
+#endif
+
+#ifdef CONFIG_RD_LZMA
+ /*
+ * If it matches the lzma magic numbers, return -1
+ */
+ if (buf[0] == 0x5d && (buf[1] == 0x00)) {
+ printk(KERN_NOTICE
+ "RAMDISK: Lzma image found at block %d\n",
+ start_block);
+ *decompressor = unlzma;
+ nblocks = 0;
+ goto done;
+ }
+#endif
/* romfs is at block zero too */
if (romfsb->word0 == ROMSB_WORD0 &&
@@ -143,6 +180,7 @@ int __init rd_load_image(char *from)
int nblocks, i, disk;
char *buf = NULL;
unsigned short rotate = 0;
+ decompress_fn decompressor = NULL;
#if !defined(CONFIG_S390) && !defined(CONFIG_PPC_ISERIES)
char rotator[4] = { '|' , '/' , '-' , '\\' };
#endif
@@ -155,12 +193,12 @@ int __init rd_load_image(char *from)
if (in_fd < 0)
goto noclose_input;
- nblocks = identify_ramdisk_image(in_fd, rd_image_start);
+ nblocks = identify_ramdisk_image(in_fd, rd_image_start, &decompressor);
if (nblocks < 0)
goto done;
if (nblocks == 0) {
- if (crd_load(in_fd, out_fd) == 0)
+ if (crd_load(in_fd, out_fd, decompressor) == 0)
goto successful_load;
goto done;
}
@@ -259,138 +297,48 @@ int __init rd_load_disk(int n)
return rd_load_image("/dev/root");
}
-/*
- * gzip declarations
- */
-
-#define OF(args) args
-
-#ifndef memzero
-#define memzero(s, n) memset ((s), 0, (n))
-#endif
-
-typedef unsigned char uch;
-typedef unsigned short ush;
-typedef unsigned long ulg;
-
-#define INBUFSIZ 4096
-#define WSIZE 0x8000 /* window size--must be a power of two, and */
- /* at least 32K for zip's deflate method */
-
-static uch *inbuf;
-static uch *window;
-
-static unsigned insize; /* valid bytes in inbuf */
-static unsigned inptr; /* index of next byte to be processed in inbuf */
-static unsigned outcnt; /* bytes in output buffer */
static int exit_code;
-static int unzip_error;
-static long bytes_out;
+static int decompress_error;
static int crd_infd, crd_outfd;
-#define get_byte() (inptr < insize ? inbuf[inptr++] : fill_inbuf())
-
-/* Diagnostic functions (stubbed out) */
-#define Assert(cond,msg)
-#define Trace(x)
-#define Tracev(x)
-#define Tracevv(x)
-#define Tracec(c,x)
-#define Tracecv(c,x)
-
-#define STATIC static
-#define INIT __init
-
-static int __init fill_inbuf(void);
-static void __init flush_window(void);
-static void __init error(char *m);
-
-#define NO_INFLATE_MALLOC
-
-#include "../lib/inflate.c"
-
-/* ===========================================================================
- * Fill the input buffer. This is called only when the buffer is empty
- * and at least one byte is really needed.
- * Returning -1 does not guarantee that gunzip() will ever return.
- */
-static int __init fill_inbuf(void)
+static int __init compr_fill(void *buf, unsigned int len)
{
- if (exit_code) return -1;
-
- insize = sys_read(crd_infd, inbuf, INBUFSIZ);
- if (insize == 0) {
- error("RAMDISK: ran out of compressed data");
- return -1;
- }
-
- inptr = 1;
-
- return inbuf[0];
+ int r = sys_read(crd_infd, buf, len);
+ if (r < 0)
+ printk(KERN_ERR "RAMDISK: error while reading compressed data");
+ else if (r == 0)
+ printk(KERN_ERR "RAMDISK: EOF while reading compressed data");
+ return r;
}
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-static void __init flush_window(void)
+static int __init compr_flush(void *window, unsigned int outcnt)
{
- ulg c = crc; /* temporary variable */
- unsigned n, written;
- uch *in, ch;
-
- written = sys_write(crd_outfd, window, outcnt);
- if (written != outcnt && unzip_error == 0) {
- printk(KERN_ERR "RAMDISK: incomplete write (%d != %d) %ld\n",
- written, outcnt, bytes_out);
- unzip_error = 1;
- }
- in = window;
- for (n = 0; n < outcnt; n++) {
- ch = *in++;
- c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
- }
- crc = c;
- bytes_out += (ulg)outcnt;
- outcnt = 0;
+ int written = sys_write(crd_outfd, window, outcnt);
+ if (written != outcnt) {
+ if (decompress_error == 0)
+ printk(KERN_ERR
+ "RAMDISK: incomplete write (%d != %d)\n",
+ written, outcnt);
+ decompress_error = 1;
+ return -1;
+ }
+ return outcnt;
}
static void __init error(char *x)
{
printk(KERN_ERR "%s\n", x);
exit_code = 1;
- unzip_error = 1;
+ decompress_error = 1;
}
-static int __init crd_load(int in_fd, int out_fd)
+static int __init crd_load(int in_fd, int out_fd, decompress_fn deco)
{
int result;
-
- insize = 0; /* valid bytes in inbuf */
- inptr = 0; /* index of next byte to be processed in inbuf */
- outcnt = 0; /* bytes in output buffer */
- exit_code = 0;
- bytes_out = 0;
- crc = (ulg)0xffffffffL; /* shift register contents */
-
crd_infd = in_fd;
crd_outfd = out_fd;
- inbuf = kmalloc(INBUFSIZ, GFP_KERNEL);
- if (!inbuf) {
- printk(KERN_ERR "RAMDISK: Couldn't allocate gzip buffer\n");
- return -1;
- }
- window = kmalloc(WSIZE, GFP_KERNEL);
- if (!window) {
- printk(KERN_ERR "RAMDISK: Couldn't allocate gzip window\n");
- kfree(inbuf);
- return -1;
- }
- makecrc();
- result = gunzip();
- if (unzip_error)
+ result = deco(NULL, 0, compr_fill, compr_flush, NULL, NULL, error);
+ if (decompress_error)
result = 1;
- kfree(inbuf);
- kfree(window);
return result;
}
diff -urNp library/init/initramfs.c generic/init/initramfs.c
--- library/init/initramfs.c 2009-01-04 11:56:42.000000000 +0100
+++ generic/init/initramfs.c 2009-01-04 21:57:13.000000000 +0100
@@ -389,11 +389,14 @@ static int __init write_buffer(char *buf
return len - count;
}
-static void __init flush_buffer(char *buf, unsigned len)
+
+static int __init flush_buffer(void *bufv, unsigned len)
{
+ char *buf = (char *) bufv;
int written;
+ int origLen = len;
if (message)
- return;
+ return -1;
while ((written = write_buffer(buf, len)) < len && !message) {
char c = buf[written];
if (c == '0') {
@@ -407,73 +410,14 @@ static void __init flush_buffer(char *bu
} else
error("junk in compressed archive");
}
+ return origLen;
}
-/*
- * gzip declarations
- */
-
-#define OF(args) args
-
-#ifndef memzero
-#define memzero(s, n) memset ((s), 0, (n))
-#endif
+static unsigned my_inptr; /* index of next byte to be processed in inbuf */
-typedef unsigned char uch;
-typedef unsigned short ush;
-typedef unsigned long ulg;
-
-#define WSIZE 0x8000 /* window size--must be a power of two, and */
- /* at least 32K for zip's deflate method */
-
-static uch *inbuf;
-static uch *window;
-
-static unsigned insize; /* valid bytes in inbuf */
-static unsigned inptr; /* index of next byte to be processed in inbuf */
-static unsigned outcnt; /* bytes in output buffer */
-static long bytes_out;
-
-#define get_byte() (inptr < insize ? inbuf[inptr++] : -1)
-
-/* Diagnostic functions (stubbed out) */
-#define Assert(cond,msg)
-#define Trace(x)
-#define Tracev(x)
-#define Tracevv(x)
-#define Tracec(c,x)
-#define Tracecv(c,x)
-
-#define STATIC static
-#define INIT __init
-
-static void __init flush_window(void);
-static void __init error(char *m);
-
-#define NO_INFLATE_MALLOC
-
-#include "../lib/inflate.c"
-
-/* ===========================================================================
- * Write the output window window[0..outcnt-1] and update crc and bytes_out.
- * (Used for the decompressed data only.)
- */
-static void __init flush_window(void)
-{
- ulg c = crc; /* temporary variable */
- unsigned n;
- uch *in, ch;
-
- flush_buffer(window, outcnt);
- in = window;
- for (n = 0; n < outcnt; n++) {
- ch = *in++;
- c = crc_32_tab[((int)c ^ ch) & 0xff] ^ (c >> 8);
- }
- crc = c;
- bytes_out += (ulg)outcnt;
- outcnt = 0;
-}
+#include <linux/decompress/bunzip2.h>
+#include <linux/decompress/unlzma.h>
+#include <linux/decompress/inflate.h>
static char * __init unpack_to_rootfs(char *buf, unsigned len, int check_only)
{
@@ -482,9 +426,10 @@ static char * __init unpack_to_rootfs(ch
header_buf = kmalloc(110, GFP_KERNEL);
symlink_buf = kmalloc(PATH_MAX + N_ALIGN(PATH_MAX) + 1, GFP_KERNEL);
name_buf = kmalloc(N_ALIGN(PATH_MAX), GFP_KERNEL);
- window = kmalloc(WSIZE, GFP_KERNEL);
- if (!window || !header_buf || !symlink_buf || !name_buf)
+
+ if (!header_buf || !symlink_buf || !name_buf)
panic("can't allocate buffers");
+
state = Start;
this_header = 0;
message = NULL;
@@ -504,22 +449,38 @@ static char * __init unpack_to_rootfs(ch
continue;
}
this_header = 0;
- insize = len;
- inbuf = buf;
- inptr = 0;
- outcnt = 0; /* bytes in output buffer */
- bytes_out = 0;
- crc = (ulg)0xffffffffL; /* shift register contents */
- makecrc();
- gunzip();
+ if (!gunzip(buf, len, NULL, flush_buffer, NULL,
+ &my_inptr, error) &&
+ message == NULL)
+ goto ok;
+
+#ifdef CONFIG_RD_BZIP2
+ message = NULL; /* Zero out message, or else cpio will
+ think an error has already occured */
+ if (!bunzip2(buf, len, NULL, flush_buffer, NULL,
+ &my_inptr, error) &&
+ message == NULL) {
+ goto ok;
+ }
+#endif
+
+#ifdef CONFIG_RD_LZMA
+ message = NULL; /* Zero out message, or else cpio will
+ think an error has already occured */
+ if (!unlzma(buf, len, NULL, flush_buffer, NULL,
+ &my_inptr, error) &&
+ message == NULL) {
+ goto ok;
+ }
+#endif
+ok:
if (state != Reset)
- error("junk in gzipped archive");
- this_header = saved_offset + inptr;
- buf += inptr;
- len -= inptr;
+ error("junk in compressed archive");
+ this_header = saved_offset + my_inptr;
+ buf += my_inptr;
+ len -= my_inptr;
}
dir_utime();
- kfree(window);
kfree(name_buf);
kfree(symlink_buf);
kfree(header_buf);
diff -urNp library/init/Kconfig generic/init/Kconfig
--- library/init/Kconfig 2009-01-04 11:56:42.000000000 +0100
+++ generic/init/Kconfig 2009-01-04 21:57:13.000000000 +0100
@@ -101,6 +101,56 @@ config LOCALVERSION_AUTO
which is done within the script "scripts/setlocalversion".)
+choice
+ prompt "Kernel compression mode"
+ default KERNEL_GZIP
+ help
+ The linux kernel is a kind of self-extracting executable.
+ Several compression algorithms are available, which differ
+ in efficiency, compression and decompression speed.
+ Compression speed is only relevant when building a kernel.
+ Decompression speed is relevant at each boot.
+
+ If you have any problems with bzip2 or lzma compressed
+ kernels, mail me (Alain Knaff) <[email protected]>. (An older
+ version of this functionality (bzip2 only), for 2.4, was
+ supplied by Christian Ludwig)
+
+ High compression options are mostly useful for users, who
+ are low on disk space (embedded systems), but for whom ram
+ size matters less.
+
+ If in doubt, select 'gzip'
+
+config KERNEL_GZIP
+ bool "Gzip"
+ help
+ The old and tried gzip compression. Its compression ratio is
+ the poorest among the 3 choices; however its speed (both
+ compression and decompression) is the fastest.
+
+config KERNEL_BZIP2
+ bool "Bzip2"
+ help
+ Its compression ratio and speed is intermediate.
+ Decompression speed is slowest among the 3.
+ The kernel size is about 10 per cent smaller with bzip2,
+ in comparison to gzip.
+ Bzip2 uses a large amount of memory. For modern kernels
+ you will need at least 8MB RAM or more for booting.
+
+config KERNEL_LZMA
+ bool "LZMA"
+ help
+ The most recent compression algorithm.
+ Its ratio is best, decompression speed is between the other
+ 2. Compression is slowest.
+ The kernel size is about 33 per cent smaller with lzma,
+ in comparison to gzip.
+
+endchoice
+
+
config SWAP
bool "Support for paging of anonymous memory (swap)"
depends on MMU && BLOCK
diff -urNp library/lib/Makefile generic/lib/Makefile
--- library/lib/Makefile 2009-01-04 11:56:42.000000000 +0100
+++ generic/lib/Makefile 2009-01-04 21:57:13.000000000 +0100
@@ -11,7 +11,8 @@ lib-y := ctype.o string.o vsprintf.o cmd
rbtree.o radix-tree.o dump_stack.o \
idr.o int_sqrt.o extable.o prio_tree.o \
sha1.o irq_regs.o reciprocal_div.o argv_split.o \
- proportions.o prio_heap.o ratelimit.o show_mem.o is_single_threaded.o
+ proportions.o prio_heap.o ratelimit.o show_mem.o is_single_threaded.o \
+ decompress_inflate.o decompress_bunzip2.o decompress_unlzma.o
lib-$(CONFIG_MMU) += ioremap.o
lib-$(CONFIG_SMP) += cpumask.o
Hi Alain,
Looking pretty good now. The only issues that I see so far are:
- Putting the kernel compression selection in init/Kconfig isn't going
to work too well, since it affects all architectures, and not all
architectures even do their own decompression (for quite a few
architectures it's the boot loader's responsibility.)
I see two options here: either have this be parameterized by the arch
Kconfig files, or simply move this chunk into arch/*/Kconfig as part
of the arch-enablement patches. The latter is probably the simplest,
even if it means some replicated code.
Let me know what you think -- I can do this pretty easily while
importing, so you don't need to submit a full new patchset.
- I did a followon patch (attached) to change the ramdisk compression
search to table driven. Since I already did the followon patch, don't
worry about it.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
H. Peter Anvin wrote:
> Hi Alain,
>
> Looking pretty good now. The only issues that I see so far are:
>
> - Putting the kernel compression selection in init/Kconfig isn't going
> to work too well, since it affects all architectures, and not all
> architectures even do their own decompression (for quite a few
> architectures it's the boot loader's responsibility.)
>
> I see two options here: either have this be parameterized by the arch
> Kconfig files, or simply move this chunk into arch/*/Kconfig as part
> of the arch-enablement patches. The latter is probably the simplest,
> even if it means some replicated code.
>
> Let me know what you think
What do you mean "parametrized by the arch Kconfig files"? Does this
mean that it would still be in the global files, but would be enabled by
some kind of switch or preprocessor macro in the arch dependant file? If
easily doable, this would IMHO be preferable.
However, if not easily feasible, I'm also ok with duplication.
> -- I can do this pretty easily while
> importing, so you don't need to submit a full new patchset.
Thanks, that would be nice.
> - I did a followon patch (attached) to change the ramdisk compression
> search to table driven. Since I already did the followon patch, don't
> worry about it.
>
> -hpa
>
>
Ok
Thanks,
Alain
Alain Knaff wrote:
>
> What do you mean "parametrized by the arch Kconfig files"? Does this
> mean that it would still be in the global files, but would be enabled by
> some kind of switch or preprocessor macro in the arch dependant file? If
> easily doable, this would IMHO be preferable.
>
Yes, that's what I mean. Basically it'd use a dependency of some sort.
It's not really clear to me right at the moment which is best, too.
Also attached another followon bug fix patch; again, you don't need to
resubmit for this.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
H. Peter Anvin wrote:
> Alain Knaff wrote:
>> What do you mean "parametrized by the arch Kconfig files"? Does this
>> mean that it would still be in the global files, but would be enabled by
>> some kind of switch or preprocessor macro in the arch dependant file? If
>> easily doable, this would IMHO be preferable.
>>
>
> Yes, that's what I mean. Basically it'd use a dependency of some sort.
> It's not really clear to me right at the moment which is best, too.
Just whatever is easiest.
> Also attached another followon bug fix patch; again, you don't need to
> resubmit for this.
>
> -hpa
>
>
Looks ok to me. Thanks,
Alain
I have pulled the x86 parts of the bzip2/lzma patchset into
tip:x86/setup-lzma. I would appreciate it if you could look at it and
make sure it looks sane. I have not added the ARM portions (patch 4),
since those should go via the ARM tree, nor the capstone patch 5, which
can only be added after the old code is removed from *all* remaining
architectures.
Ingo: this *will* break your randconfig testing if you have machines
where the lzma(1) utility isn't installed!
The shortlog looks like:
Alain Knaff (3):
bzip2/lzma: library support for gzip, bzip2 and lzma decompression
bzip2/lzma: config and initramfs support for bzip2/lzma decompression
bzip2/lzma: x86 kernel compression support
H. Peter Anvin (3):
bzip2/lzma: use a table to search for initramfs compression formats
bzip2/lzma: handle failures from bzip2 and lzma correctly
bzip2/lzma: make config machinery an arch configurable
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Sun, Jan 04, 2009 at 03:58:23PM -0800, H. Peter Anvin wrote:
> I have pulled the x86 parts of the bzip2/lzma patchset into
> tip:x86/setup-lzma. I would appreciate it if you could look at it and
> make sure it looks sane. I have not added the ARM portions (patch 4),
> since those should go via the ARM tree, nor the capstone patch 5, which
> can only be added after the old code is removed from *all* remaining
> architectures.
>
> Ingo: this *will* break your randconfig testing if you have machines
> where the lzma(1) utility isn't installed!
>
> The shortlog looks like:
>
> Alain Knaff (3):
> bzip2/lzma: library support for gzip, bzip2 and lzma decompression
> bzip2/lzma: config and initramfs support for bzip2/lzma decompression
> bzip2/lzma: x86 kernel compression support
>
> H. Peter Anvin (3):
> bzip2/lzma: use a table to search for initramfs compression formats
> bzip2/lzma: handle failures from bzip2 and lzma correctly
> bzip2/lzma: make config machinery an arch configurable
Can you post this last patch.
Thanks,
Sam
Sam Ravnborg wrote:
> On Sun, Jan 04, 2009 at 03:58:23PM -0800, H. Peter Anvin wrote:
>> I have pulled the x86 parts of the bzip2/lzma patchset into
>> tip:x86/setup-lzma. I would appreciate it if you could look at it and
>> make sure it looks sane. I have not added the ARM portions (patch 4),
>> since those should go via the ARM tree, nor the capstone patch 5, which
>> can only be added after the old code is removed from *all* remaining
>> architectures.
>>
>> Ingo: this *will* break your randconfig testing if you have machines
>> where the lzma(1) utility isn't installed!
>>
>> The shortlog looks like:
>>
>> Alain Knaff (3):
>> bzip2/lzma: library support for gzip, bzip2 and lzma decompression
>> bzip2/lzma: config and initramfs support for bzip2/lzma decompression
>> bzip2/lzma: x86 kernel compression support
>>
>> H. Peter Anvin (3):
>> bzip2/lzma: use a table to search for initramfs compression formats
>> bzip2/lzma: handle failures from bzip2 and lzma correctly
>> bzip2/lzma: make config machinery an arch configurable
>
> Can you post this last patch.
>
> Thanks,
> Sam
Here it is.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Sun, Jan 04, 2009 at 09:09:21PM -0800, H. Peter Anvin wrote:
> Sam Ravnborg wrote:
> > On Sun, Jan 04, 2009 at 03:58:23PM -0800, H. Peter Anvin wrote:
> >> I have pulled the x86 parts of the bzip2/lzma patchset into
> >> tip:x86/setup-lzma. I would appreciate it if you could look at it and
> >> make sure it looks sane. I have not added the ARM portions (patch 4),
> >> since those should go via the ARM tree, nor the capstone patch 5, which
> >> can only be added after the old code is removed from *all* remaining
> >> architectures.
> >>
> >> Ingo: this *will* break your randconfig testing if you have machines
> >> where the lzma(1) utility isn't installed!
> >>
> >> The shortlog looks like:
> >>
> >> Alain Knaff (3):
> >> bzip2/lzma: library support for gzip, bzip2 and lzma decompression
> >> bzip2/lzma: config and initramfs support for bzip2/lzma decompression
> >> bzip2/lzma: x86 kernel compression support
> >>
> >> H. Peter Anvin (3):
> >> bzip2/lzma: use a table to search for initramfs compression formats
> >> bzip2/lzma: handle failures from bzip2 and lzma correctly
> >> bzip2/lzma: make config machinery an arch configurable
> >
> > Can you post this last patch.
> >
> > Thanks,
> > Sam
>
> Here it is.
Looks good.
You can add a:
Reviewed-by: Sam Ravnborg <[email protected]>
if you like.
Sam
H. Peter Anvin wrote:
> Alain Knaff wrote:
>>
>> Yes, now I remember. We had that subject of compressed builtin
>> initramfs' a while ago. Back then, I proposed to just leave it
>> uncompressed (considering that it is part of the kernel, which is
>> compressed anyways)... but that apparently caused problems for some
>> people who liked to store huge amount in there, due to some artifacts
>> about how initramfs initialization works.
>>
>> Well, I'd think in that case, the most expedient method would be to have
>> CONFIG_RD_GZIP always defined.
>>
>
> Expedient, perhaps, but it's not the right solution. The same people
> who want a large builtin initramfs are generally the ones that care
> about image size. If we gzip the internal image, it will in fact
> nullify the benefit of using a better compressor for the kernel image in
> general, since the gzipped portion of the image will hardly compress at
> all.
>
> -hpa
Exactly the reason why in one of my first versions I wanted to do away
with that feature.... until you pointed out that it was actually using
more *memory* when not compressed, because of the way the buffer cache
works when the initramfs is put into place (it is not freed "as we go",
but only at the very end, so at one point in time there will be 2 copies
which will of course be larger when they're both uncompressed).
I'm getting the impression that again, anybody wanting to propose an
enhancement must solve _all_ pre-existing problems in that area... While
this certainly leads to better code once a patch does make it, it will
certainly discourage many.
Maybe we can separate both issues, and tackle the initramfs case
entirely separately (maybe by proving a buffer cache enhancement that
allows to free initramfs' memory "as we go").
Regards,
Alain
Alain Knaff wrote:
> Exactly the reason why in one of my first versions I wanted to do away
> with that feature.... until you pointed out that it was actually using
> more *memory* when not compressed, because of the way the buffer cache
> works when the initramfs is put into place (it is not freed "as we go",
> but only at the very end, so at one point in time there will be 2 copies
> which will of course be larger when they're both uncompressed).
>
> I'm getting the impression that again, anybody wanting to propose an
> enhancement must solve _all_ pre-existing problems in that area... While
> this certainly leads to better code once a patch does make it, it will
> certainly discourage many.
Yes. That is a deliberate tradeoff in our culture. It's hardly a
secret, either.
> Maybe we can separate both issues, and tackle the initramfs case
> entirely separately (maybe by proving a buffer cache enhancement that
> allows to free initramfs' memory "as we go").
Well, I think the right thing to do at this stage is to simply compress
the initramfs with the preferred compression method (and if no
compression method is provided, with none.) This can be relatively
simply done with a script, I think.
At least that way the whole image is compressed using the preferred
compression format, and since all the relevant compressors have methods
of dealing reasonably with uncompressable chunks we should be okay.
Doing the "free as you go" thing is definitely something I would like to
see, as well as supporting initramfs in high memory. I have looked at
it a few times, however, it is a nontrivial modification and I haven't
personally devoted the time to dealing with it. I really don't think we
want to entangle this issue with the compression patch.
-hpa
H. Peter Anvin wrote:
> Well, I think the right thing to do at this stage is to simply compress
> the initramfs with the preferred compression method (and if no
> compression method is provided, with none.) This can be relatively
> simply done with a script, I think.
Well, the problem is there is really no "preferred" method for the
ramdisk or initramfs. And picking the one from the kernel might not
work, because the user might have chosen to compress the kernel via
bzip2, but only allowed lzma and gzip for the ramdisk.
> At least that way the whole image is compressed using the preferred
> compression format, and since all the relevant compressors have methods
> of dealing reasonably with uncompressable chunks we should be okay.
>
> Doing the "free as you go" thing is definitely something I would like to
> see, as well as supporting initramfs in high memory. I have looked at
> it a few times, however, it is a nontrivial modification and I haven't
> personally devoted the time to dealing with it. I really don't think we
> want to entangle this issue with the compression patch.
>
> -hpa
Agreed
Alain
Alain Knaff wrote:
> H. Peter Anvin wrote:
>> Well, I think the right thing to do at this stage is to simply compress
>> the initramfs with the preferred compression method (and if no
>> compression method is provided, with none.) This can be relatively
>> simply done with a script, I think.
>
> Well, the problem is there is really no "preferred" method for the
> ramdisk or initramfs. And picking the one from the kernel might not
> work, because the user might have chosen to compress the kernel via
> bzip2, but only allowed lzma and gzip for the ramdisk.
>
I'm thinking that with "preferred" we might just use the first one in
the priority list "lzma, bzip2, gzip, uncompressed" depending on what is
installed in the kernel.
Using the kernel method may definitely not work, especially on
architectures which don't do this style of kernel compression.
-hpa
H. Peter Anvin wrote:
> I'm thinking that with "preferred" we might just use the first one in
> the priority list "lzma, bzip2, gzip, uncompressed" depending on what is
> installed in the kernel.
>
> Using the kernel method may definitely not work, especially on
> architectures which don't do this style of kernel compression.
>
> -hpa
Ok. For the new change, may I do a diff against the current
x86/lzma-setup , or should I do the whole 5-part dance again?
Thx,
Alain
Alain Knaff wrote:
> H. Peter Anvin wrote:
>> I'm thinking that with "preferred" we might just use the first one in
>> the priority list "lzma, bzip2, gzip, uncompressed" depending on what is
>> installed in the kernel.
>>
>> Using the kernel method may definitely not work, especially on
>> architectures which don't do this style of kernel compression.
>>
>> -hpa
>
> Ok. For the new change, may I do a diff against the current
> x86/lzma-setup , or should I do the whole 5-part dance again?
>
Go ahead and do an incremental patch at this point.
-hpa
H. Peter Anvin wrote:
> Go ahead and do an incremental patch at this point.
>
> -hpa
Thanks, that'll make things easier. Unfortunately, I can no longer check
out the code. No error messages, but the files just aren't there:
> git pull
Already up-to-date.
> git checkout -b x86/setup-lzma tip/x86/setup-lzma
git checkout: branch x86/setup-lzma already exists
> ls lib/decompress_inflate.c
ls: cannot access lib/decompress_inflate.c: No such file or directory
Thanks
Alain
On Tue, Jan 6, 2009 at 3:41 AM, H. Peter Anvin <[email protected]> wrote:
>
> I'm thinking that with "preferred" we might just use the first one in the
> priority list "lzma, bzip2, gzip, uncompressed" depending on what is
> installed in the kernel.
>
Only problem I faced with lzma is that, in some machines /usr/bin/lzma
is not installed by default which leads to error in kernel
compilations.
Can we check in kernel config that lzma is installed on machine or not.
--
JSR
Alain Knaff wrote:
> H. Peter Anvin wrote:
>> Go ahead and do an incremental patch at this point.
>>
>> -hpa
>
> Thanks, that'll make things easier. Unfortunately, I can no longer check
> out the code. No error messages, but the files just aren't there:
>
>
>> git pull
> Already up-to-date.
>> git checkout -b x86/setup-lzma tip/x86/setup-lzma
> git checkout: branch x86/setup-lzma already exists
I think the "git checkout" failed here because you were trying to create
a local branch (-b) failed. Just "git checkout x86/setup-lzma" followed
by "git pull" *should* work...
>> ls lib/decompress_inflate.c
> ls: cannot access lib/decompress_inflate.c: No such file or directory
>
If that doesn't work, you may want to ask the people in #git on Freenode.
-hpa
On Tue, Jan 06, 2009 at 08:09:17AM +0100, Alain Knaff wrote:
> H. Peter Anvin wrote:
> > Go ahead and do an incremental patch at this point.
> >
> > -hpa
>
> Thanks, that'll make things easier. Unfortunately, I can no longer check
> out the code. No error messages, but the files just aren't there:
>
>
> > git pull
> Already up-to-date.
run "git branch" to see what current branch you're working on.
> > git checkout -b x86/setup-lzma tip/x86/setup-lzma
> git checkout: branch x86/setup-lzma already exists
you should just to "git checkout x86/setup-lzma" since you have already
created that branch (-b XXX YYY creates branch XXX from YYY before doing
the checkout).
normally, if you do the checkout first, then the pull, you should be OK.
Regards,
Willy
Jaswinder Singh Rajput wrote:
> On Tue, Jan 6, 2009 at 3:41 AM, H. Peter Anvin <[email protected]> wrote:
>> I'm thinking that with "preferred" we might just use the first one in the
>> priority list "lzma, bzip2, gzip, uncompressed" depending on what is
>> installed in the kernel.
>
> Only problem I faced with lzma is that, in some machines /usr/bin/lzma
> is not installed by default which leads to error in kernel
> compilations.
>
> Can we check in kernel config that lzma is installed on machine or not.
For building the default initramfs, we'd do it in a script. I don't
think there is any way to check the system for configuration options,
and I would argue that it *shouldn't* be, because it creates a
silent failure condition.
-hpa
H. Peter Anvin wrote:
> Alain Knaff wrote:
>> H. Peter Anvin wrote:
>>> Go ahead and do an incremental patch at this point.
>>>
>>> -hpa
>>
>> Thanks, that'll make things easier. Unfortunately, I can no longer check
>> out the code. No error messages, but the files just aren't there:
>>
>>
>>> git pull
>> Already up-to-date.
>>> git checkout -b x86/setup-lzma tip/x86/setup-lzma
>> git checkout: branch x86/setup-lzma already exists
>
> I think the "git checkout" failed here because you were trying to create
> a local branch (-b) failed. Just "git checkout x86/setup-lzma" followed
> by "git pull" *should* work...
Yes, that did indeed work. Thanks
> git checkout x86/setup-lzma
Switched to branch "x86/setup-lzma"
> git pull
Updating 2e9f3bd..c8531ab
Fast forward
drivers/block/Kconfig | 20 +++++++++++---------
lib/Kconfig | 13 +++++++++++++
lib/Makefile | 7 +++++--
3 files changed, 29 insertions(+), 11 deletions(-)
> ls lib/decompress_inflate.c
lib/decompress_inflate.c
Alain
On Tue, Jan 6, 2009 at 12:54 PM, H. Peter Anvin <[email protected]> wrote:
> Jaswinder Singh Rajput wrote:
>>
>> On Tue, Jan 6, 2009 at 3:41 AM, H. Peter Anvin <[email protected]> wrote:
>>>
>>> I'm thinking that with "preferred" we might just use the first one in the
>>> priority list "lzma, bzip2, gzip, uncompressed" depending on what is
>>> installed in the kernel.
>>
>> Only problem I faced with lzma is that, in some machines /usr/bin/lzma
>> is not installed by default which leads to error in kernel
>> compilations.
>>
>> Can we check in kernel config that lzma is installed on machine or not.
>
> For building the default initramfs, we'd do it in a script. I don't think
> there is any way to check the system for configuration options, and I would
> argue that it *shouldn't* be, because it creates a
> silent failure condition.
Sorry, I was talking about kernel with CONFIG_KERNEL_LZMA=y
On some machines where /usr/bin/lzma was not installed I was getting
errors during kernel compilation. So I installed lzma package and then
kernel compiled for me on that machine.
--
JSR
Jaswinder Singh Rajput wrote:
>>>
>>> Can we check in kernel config that lzma is installed on machine or not.
>> For building the default initramfs, we'd do it in a script. I don't think
>> there is any way to check the system for configuration options, and I would
>> argue that it *shouldn't* be, because it creates a
>> silent failure condition.
>
> Sorry, I was talking about kernel with CONFIG_KERNEL_LZMA=y
>
> On some machines where /usr/bin/lzma was not installed I was getting
> errors during kernel compilation. So I installed lzma package and then
> kernel compiled for me on that machine.
>
Right, well, that's to be expected. (There was a bug earlier on which
would cause it to "complete" incorrectly... not good for obvious reasons.)
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
H. Peter Anvin wrote:
> Alain Knaff wrote:
>> H. Peter Anvin wrote:
>>> I'm thinking that with "preferred" we might just use the first one in
>>> the priority list "lzma, bzip2, gzip, uncompressed" depending on what is
>>> installed in the kernel.
>>>
>>> Using the kernel method may definitely not work, especially on
>>> architectures which don't do this style of kernel compression.
>>>
>>> -hpa
>>
>> Ok. For the new change, may I do a diff against the current
>> x86/lzma-setup , or should I do the whole 5-part dance again?
>>
>
> Go ahead and do an incremental patch at this point.
>
> -hpa
Here it is (see attachment)
Alain
Alain Knaff wrote:
> It also restores the select ZLIB_INFLATE in drivers/block/Kconfig
> which somehow came missing. This is needed to activate compilation of
> the stuff in zlib_deflate.
It's not missing. It is (or at least should be) included via the
dependency chain DECOMPRESS_GZIP -> ZLIB_INFLATE.
-hpa
H. Peter Anvin wrote:
> Alain Knaff wrote:
>> It also restores the select ZLIB_INFLATE in drivers/block/Kconfig
>> which somehow came missing. This is needed to activate compilation of
>> the stuff in zlib_deflate.
>
> It's not missing. It is (or at least should be) included via the
> dependency chain DECOMPRESS_GZIP -> ZLIB_INFLATE.
>
> -hpa
Well, that didn't work when I tried it here...
Alain
Alain Knaff wrote:
> H. Peter Anvin wrote:
>> Alain Knaff wrote:
>>> It also restores the select ZLIB_INFLATE in drivers/block/Kconfig
>>> which somehow came missing. This is needed to activate compilation of
>>> the stuff in zlib_deflate.
>> It's not missing. It is (or at least should be) included via the
>> dependency chain DECOMPRESS_GZIP -> ZLIB_INFLATE.
>>
>> -hpa
>
> Well, that didn't work when I tried it here...
>
OK, let's fix the dependency chain then. I'll mess with it.
-hpa
H. Peter Anvin wrote:
> Alain Knaff wrote:
>> H. Peter Anvin wrote:
>>> Alain Knaff wrote:
>>>> It also restores the select ZLIB_INFLATE in drivers/block/Kconfig
>>>> which somehow came missing. This is needed to activate compilation of
>>>> the stuff in zlib_deflate.
>>> It's not missing. It is (or at least should be) included via the
>>> dependency chain DECOMPRESS_GZIP -> ZLIB_INFLATE.
>>>
>>> -hpa
>>
>> Well, that didn't work when I tried it here...
>>
>
> OK, let's fix the dependency chain then. I'll mess with it.
>
> -hpa
Ok, go ahead.
Alain
Alain Knaff wrote:
> Alain Knaff wrote:
>> hmmm, but on a second glance, there seems to be indeed an issue. When
>> compiling it a _second time_, compilation will "succeed" but produce a
>> bad kernel because the first attempt left an empty
>> usr/initramfs_data.cpio.bz2 .
>>
>> I'll look into it this evening, probably a case of "bad" failure
>> recovery in the Makefile.
>>
>
> Took me slightly longer than promised, attached is the patch.
>
> It changes/fixes three things:
> 1. Fix a bug in decompress.c : only scanned until the first
> non-configured compressor (with disastrous result if that was gzip)
> 2. Fix a bug in gen_initramfs_list.sh : in case of failure, it left
> indeed an empty output file behind, messing up the next make.
> 3. Make builtin initramfs compression configurable
>
> Regards,
>
> Alain
>
Any comments about this? Ingo, does it solve the issue with randconfig?
Btw, when will the next merge-window be, so that I can plan ahead to be
available enough?
Thanks,
Alain
Alain Knaff wrote:
>
> Any comments about this? Ingo, does it solve the issue with randconfig?
>
> Btw, when will the next merge-window be, so that I can plan ahead to be
> available enough?
>
Noone knows but Linus. However, we're at -rc6 now, so it seems logical
that it might be in 1-2 weeks time.
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
* Alain Knaff <[email protected]> wrote:
> Alain Knaff wrote:
> > Alain Knaff wrote:
> >> hmmm, but on a second glance, there seems to be indeed an issue. When
> >> compiling it a _second time_, compilation will "succeed" but produce a
> >> bad kernel because the first attempt left an empty
> >> usr/initramfs_data.cpio.bz2 .
> >>
> >> I'll look into it this evening, probably a case of "bad" failure
> >> recovery in the Makefile.
> >>
> >
> > Took me slightly longer than promised, attached is the patch.
> >
> > It changes/fixes three things:
> > 1. Fix a bug in decompress.c : only scanned until the first
> > non-configured compressor (with disastrous result if that was gzip)
> > 2. Fix a bug in gen_initramfs_list.sh : in case of failure, it left
> > indeed an empty output file behind, messing up the next make.
> > 3. Make builtin initramfs compression configurable
> >
> > Regards,
> >
> > Alain
> >
>
> Any comments about this? Ingo, does it solve the issue with
> randconfig?
I've reactivated tip:x86/setup-lzma in tip:master and started
testing it with the problem .config, on the problem test-system.
The bug is most likely fixed, so consider it queued up for
2.6.30 unless i report some new issue.
Thanks,
Ingo
Ingo Molnar wrote:
> * Alain Knaff <[email protected]> wrote:
>> Any comments about this? Ingo, does it solve the issue with
>> randconfig?
>
> I've reactivated tip:x86/setup-lzma in tip:master and started
> testing it with the problem .config, on the problem test-system.
> The bug is most likely fixed, so consider it queued up for
> 2.6.30 unless i report some new issue.
>
> Thanks,
>
> Ingo
Thanks. I'm glad we finally made it :-)
Regards,
Alain
* Alain Knaff <[email protected]> wrote:
> Ingo Molnar wrote:
> > * Alain Knaff <[email protected]> wrote:
> >> Any comments about this? Ingo, does it solve the issue with
> >> randconfig?
> >
> > I've reactivated tip:x86/setup-lzma in tip:master and started
> > testing it with the problem .config, on the problem test-system.
> > The bug is most likely fixed, so consider it queued up for
> > 2.6.30 unless i report some new issue.
> >
> > Thanks,
> >
> > Ingo
>
> Thanks. I'm glad we finally made it :-)
Yeah, it was quite unusually difficult. Probably because this is
dusty but still very central code - it was always there and
always worked - so if this breaks then everyone is up in arms.
Ingo