From dd997b4fb8c410e6b36f6b7feece72e0c8228828 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 25 Aug 2026 04:48:29 +0900 Subject: [PATCH 1/9] test-compress: compare decompressed data with source --- src/test/test-compress.c | 49 +++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/test/test-compress.c b/src/test/test-compress.c index bfffcc4804..ac7f2e5415 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -417,6 +417,29 @@ static int test_decompressor_callback(const void *p, size_t size, void *userdata return 0; } +static void compare_fd(int fda, int fdb) { + struct stat sta, stb; + + /* Verify apparent size matches */ + ASSERT_OK_ERRNO(fstat(fda, &sta)); + ASSERT_OK_ERRNO(fstat(fdb, &stb)); + ASSERT_EQ(sta.st_size, stb.st_size); + + /* Verify content matches by comparing bytes */ + ASSERT_OK_EQ_ERRNO(lseek(fda, 0, SEEK_SET), (off_t) 0); + ASSERT_OK_EQ_ERRNO(lseek(fdb, 0, SEEK_SET), (off_t) 0); + + for (off_t offset = 0; offset < sta.st_size;) { + uint8_t bufa[4096], bufb[4096]; + size_t to_read = MIN((size_t) (sta.st_size - offset), sizeof(bufa)); + + ASSERT_OK_EQ(loop_read(fda, bufa, to_read, true), (ssize_t) to_read); + ASSERT_OK_EQ(loop_read(fdb, bufb, to_read, true), (ssize_t) to_read); + ASSERT_EQ(memcmp(bufa, bufb, to_read), 0); + offset += to_read; + } +} + TEST(decompress_stream_sparse) { for (Compression c = 0; c < _COMPRESSION_MAX; c++) { if (c == COMPRESSION_NONE || !compression_supported(c)) @@ -460,24 +483,9 @@ TEST(decompress_stream_sparse) { ASSERT_EQ(lseek(compressed, 0, SEEK_SET), (off_t) 0); ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size)); - /* Verify apparent size matches */ + compare_fd(src, decompressed); + ASSERT_OK_ERRNO(fstat(decompressed, &st_decompressed)); - ASSERT_EQ(st_decompressed.st_size, st_src.st_size); - - /* Verify content matches by comparing bytes */ - ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0); - ASSERT_EQ(lseek(decompressed, 0, SEEK_SET), (off_t) 0); - - for (off_t offset = 0; offset < st_src.st_size;) { - uint8_t buf_src[4096], buf_dst[4096]; - size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src)); - - ASSERT_EQ(loop_read(src, buf_src, to_read, true), (ssize_t) to_read); - ASSERT_EQ(loop_read(decompressed, buf_dst, to_read, true), (ssize_t) to_read); - ASSERT_EQ(memcmp(buf_src, buf_dst, to_read), 0); - offset += to_read; - } - /* Verify the decompressed file is actually sparse (uses less disk than apparent size). * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be * noticeably less than the apparent size if sparse writes worked. @@ -515,8 +523,9 @@ TEST(decompress_stream_sparse) { ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0); ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, sizeof(zeros))); + compare_fd(zsrc, zdecompressed); + ASSERT_OK_ERRNO(fstat(zdecompressed, &zst)); - ASSERT_EQ(zst.st_size, (off_t) sizeof(zeros)); /* All zeros — disk usage should be minimal */ log_debug("%s all-zeros sparse: apparent=%jd disk=%jd", compression_to_string(c), (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512); @@ -534,7 +543,6 @@ TEST(decompress_stream_sparse) { dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX", dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; - struct stat dst; uint64_t dsize; uint8_t zeros[65536] = {}; @@ -552,8 +560,7 @@ TEST(decompress_stream_sparse) { ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0); ASSERT_OK_ZERO(decompress_stream(c, dcompressed, ddecompressed, dsize)); - ASSERT_OK_ERRNO(fstat(ddecompressed, &dst)); - ASSERT_EQ(dst.st_size, (off_t)(sizeof(zeros) + sizeof(data_block))); + compare_fd(dsrc, ddecompressed); } } } From fb1160c58836a6dc23c91af0af60be5447b49900 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 27 Aug 2026 01:01:55 +0900 Subject: [PATCH 2/9] test-compress: replace writing zeros with ftruncate() + lseek() --- src/test/test-compress.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/test/test-compress.c b/src/test/test-compress.c index ac7f2e5415..ec28d6f294 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -508,20 +508,19 @@ TEST(decompress_stream_sparse) { zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX", zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX"; struct stat zst; - uint64_t zsize; - uint8_t zeros[65536] = {}; + uint64_t zsize, zsize_expected = 65536; ASSERT_OK(zsrc = mkostemp_safe(zp_src)); - ASSERT_OK(loop_write(zsrc, zeros, sizeof(zeros))); + ASSERT_OK_ERRNO(ftruncate(zsrc, zsize_expected)); ASSERT_EQ(lseek(zsrc, 0, SEEK_SET), (off_t) 0); ASSERT_OK(zcompressed = mkostemp_safe(zp_compressed)); ASSERT_OK(compress_stream(c, zsrc, zcompressed, -1, &zsize)); - ASSERT_EQ(zsize, (uint64_t) sizeof(zeros)); + ASSERT_EQ(zsize, zsize_expected); ASSERT_OK(zdecompressed = mkostemp_safe(zp_decompressed)); ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0); - ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, sizeof(zeros))); + ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, zsize_expected)); compare_fd(zsrc, zdecompressed); @@ -544,17 +543,17 @@ TEST(decompress_stream_sparse) { dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; uint64_t dsize; - uint8_t zeros[65536] = {}; /* 64K zeros followed by 4K random data */ ASSERT_OK(dsrc = mkostemp_safe(dp_src)); - ASSERT_OK(loop_write(dsrc, zeros, sizeof(zeros))); + ASSERT_OK_ERRNO(ftruncate(dsrc, 65536)); + ASSERT_OK_ERRNO(lseek(dsrc, 65536, SEEK_SET)); ASSERT_OK(loop_write(dsrc, data_block, sizeof(data_block))); ASSERT_EQ(lseek(dsrc, 0, SEEK_SET), (off_t) 0); ASSERT_OK(dcompressed = mkostemp_safe(dp_compressed)); ASSERT_OK(compress_stream(c, dsrc, dcompressed, -1, &dsize)); - ASSERT_EQ(dsize, (uint64_t)(sizeof(zeros) + sizeof(data_block))); + ASSERT_EQ(dsize, (uint64_t) (65536 + sizeof(data_block))); ASSERT_OK(ddecompressed = mkostemp_safe(dp_decompressed)); ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0); From d365c4cd52a25bf2164c2abb8bd02ef6d0a1fe5e Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 27 Aug 2026 01:32:17 +0900 Subject: [PATCH 3/9] compress: fix error handling for lzma_lzma_preset() Follow-up for 91d6f1ee5336f93369d21af8e17ca3159ae46cff. --- src/basic/compress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index 8cbf22005d..6747d065c6 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -482,9 +482,9 @@ static int compress_blob_xz( return r; if (level >= 0) { - r = sym_lzma_lzma_preset(&opt, (uint32_t) level); - if (r < 0) - return r; + /* This returns lzma_bool, and true means failure, i.e. the specified level is not supported. */ + if (sym_lzma_lzma_preset(&opt, (uint32_t) level)) + return -EINVAL; } /* Returns < 0 if we couldn't compress the data or the From b9dbbccd201404d33cafa2456c69a1eb0f512c2e Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 27 Aug 2026 01:41:23 +0900 Subject: [PATCH 4/9] compress: fix indentation and add short comment --- src/basic/compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index 6747d065c6..bcafec69b0 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -493,8 +493,8 @@ static int compress_blob_xz( if (src_size < 80) return -ENOBUFS; - ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL, - src, src_size, dst, &out_pos, dst_alloc_size); + ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, /* allocator= */ NULL, + src, src_size, dst, &out_pos, dst_alloc_size); if (ret != LZMA_OK) return -ENOBUFS; From fefe8f3132492805db3f64180f720eb1ff295525 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 27 Aug 2026 04:36:26 +0900 Subject: [PATCH 5/9] compress: use consistent error code for failure in BZ2_bzDecompressInit() --- src/basic/compress.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index bcafec69b0..170282a9f2 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -1040,9 +1040,8 @@ static int decompress_blob_bzip2( _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; - r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) - return -ENOMEM; + if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX); if (!greedy_realloc(dst, space, 1)) @@ -1477,9 +1476,8 @@ static int decompress_startswith_bzip2( _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; - r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) - return -EBADMSG; + if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1))) return -ENOMEM; From f577c38a5261cda98a8f8998fbf1905be0f416cc Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Thu, 27 Aug 2026 04:36:50 +0900 Subject: [PATCH 6/9] compress: inline error handling --- src/basic/compress.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index 170282a9f2..b741f3db2f 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -1700,10 +1700,6 @@ static int decompress_stream_write_callback(const void *data, size_t size, void } static int decompressor_new(Decompressor **ret, Compression type) { -#if HAVE_ZLIB || HAVE_BZIP2 - int r; -#endif - assert(ret); _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1); @@ -1743,16 +1739,14 @@ static int decompressor_new(Decompressor **ret, Compression type) { #if HAVE_ZLIB case COMPRESSION_GZIP: - r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)); - if (r != Z_OK) + if (sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)) != Z_OK) return -EIO; break; #endif #if HAVE_BZIP2 case COMPRESSION_BZIP2: - r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) + if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) return -EIO; break; #endif From d9b6e8663c030b8e52e6523c71bb5e175fad00a2 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 25 Aug 2026 04:22:52 +0900 Subject: [PATCH 7/9] compress: check cursor position in should_sparse() Otherwise, the result will be broken, as skipped area by lseek() is not cleared. --- src/basic/compress.c | 19 ++++++++++++++++--- src/test/test-compress.c | 25 +++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index b741f3db2f..eeecd59b41 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -1632,21 +1632,34 @@ int compress_stream( return 0; } -/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on - * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */ static int should_sparse(int fd) { struct stat st; assert(fd >= 0); + /* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on regular + * files, without O_APPEND (which ignores the lseek position, collapsing any holes we try to create), + * and only when starting from a position at or beyond the current end of file (otherwise the skipped + * ranges would overlap pre-existing data instead of being genuinely unwritten). */ + if (fstat(fd, &st) < 0) return -errno; + if (!S_ISREG(st.st_mode)) + return false; + int flags = fcntl(fd, F_GETFL); if (flags < 0) return -errno; - return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND); + if (FLAGS_SET(flags, O_APPEND)) + return false; + + off_t pos = lseek(fd, 0, SEEK_CUR); + if (pos < 0) + return -errno; + + return pos >= st.st_size; } /* After sparse decompression, set the file size to the current position to account for diff --git a/src/test/test-compress.c b/src/test/test-compress.c index ec28d6f294..6ab482f718 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -433,8 +433,8 @@ static void compare_fd(int fda, int fdb) { uint8_t bufa[4096], bufb[4096]; size_t to_read = MIN((size_t) (sta.st_size - offset), sizeof(bufa)); - ASSERT_OK_EQ(loop_read(fda, bufa, to_read, true), (ssize_t) to_read); - ASSERT_OK_EQ(loop_read(fdb, bufb, to_read, true), (ssize_t) to_read); + ASSERT_OK(loop_read_exact(fda, bufa, to_read, /* do_poll= */ false)); + ASSERT_OK(loop_read_exact(fdb, bufb, to_read, /* do_poll= */ false)); ASSERT_EQ(memcmp(bufa, bufb, to_read), 0); offset += to_read; } @@ -499,6 +499,27 @@ TEST(decompress_stream_sparse) { else log_debug("Filesystem does not support holes, skipping sparsity check"); + /* Check if the decompressor clears previous data. + * The original data is 4K data, 64K zeros, 4K data, 64K zeros. + * Now, fill the file with 4K zeros, 64K data, 4K zeros, 64K data. */ + log_debug("/* testing %s sparse decompression to an existing file */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(decompressed, 0)); + ASSERT_OK_ERRNO(ftruncate(decompressed, 4096)); + ASSERT_OK_ERRNO(lseek(decompressed, 4096, SEEK_SET)); + for (unsigned i = 0; i < 16; i++) + ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block))); + ASSERT_OK_ERRNO(ftruncate(decompressed, 4096 + 16 * sizeof(data_block) + 4096)); + ASSERT_OK_ERRNO(lseek(decompressed, 4096 + 16 * sizeof(data_block) + 4096, SEEK_SET)); + for (unsigned i = 0; i < 16; i++) + ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block))); + + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size)); + + compare_fd(src, decompressed); + /* Test all-zeros input: entire output should be a hole */ log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c)); { From dac2b39405cd3a8bfad8d624202a7e03cdf6c7aa Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Tue, 25 Aug 2026 13:15:07 +0900 Subject: [PATCH 8/9] compress: strictly check if stream finished on decompress Previously, even when the compressed data was truncated, or had trailing garbage, the decompressor might not detect it. To fix the issue, update the check if stream is finished. With this change, now decompressor_push() must be called with size == 0 to finalize the stream. --- src/basic/compress.c | 195 +++++++++++++++++++++++++++++++-------- src/test/test-compress.c | 59 +++++++++++- 2 files changed, 211 insertions(+), 43 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index eeecd59b41..f3e57dd284 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -133,6 +133,7 @@ static DLSYM_PROTOTYPE(deflateEnd) = NULL; static DLSYM_PROTOTYPE(inflateInit2_) = NULL; static DLSYM_PROTOTYPE(inflate) = NULL; static DLSYM_PROTOTYPE(inflateEnd) = NULL; +static DLSYM_PROTOTYPE(inflateReset) = NULL; static inline void deflateEnd_wrapper(z_stream *s) { sym_deflateEnd(s); @@ -164,6 +165,7 @@ static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) { struct Compressor { Compression type; bool encoding; + bool finished; union { #if HAVE_XZ lzma_stream xz; @@ -396,7 +398,8 @@ int dlopen_zlib(int log_level) { DLSYM_ARG(deflateEnd), DLSYM_ARG(inflateInit2_), DLSYM_ARG(inflate), - DLSYM_ARG(inflateEnd)); + DLSYM_ARG(inflateEnd), + DLSYM_ARG(inflateReset)); #else return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP), "zlib support is not compiled in."); @@ -1814,14 +1817,15 @@ int decompress_stream( n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE); if (n < 0) return -errno; - if (n == 0) - break; total_in += n; r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata); if (r < 0) return r; + + if (n == 0) + break; } if (total_in == 0) @@ -1958,15 +1962,20 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress int r; assert(c); + assert(data || size == 0); assert(callback); + /* This feeds `size` bytes of compressed data into `c`. Passing size == 0 has special meaning: it + * signals that no further input will be provided, and the function will attempt to finalize the + * stream, failing with -EBADMSG if the compressed data was truncated or has trailing garbage after + * the logical end of stream. */ + if (c->encoding) return -EINVAL; - if (size == 0) - return 1; - - assert(data); + if (c->finished && size == 0) + /* The previous stream was correctly finished, and it was the last one. */ + return 0; #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 _cleanup_free_ uint8_t *buffer = NULL; @@ -1980,33 +1989,61 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress switch (c->type) { case COMPRESSION_NONE: + if (c->finished) + return -EBADMSG; + + if (size == 0) { + c->finished = true; + return 0; + } + r = callback(data, size, userdata); if (r < 0) return r; - break; + return 0; #if HAVE_XZ case COMPRESSION_XZ: + if (c->finished) + return -EBADMSG; + c->xz.next_in = data; c->xz.avail_in = size; - while (c->xz.avail_in > 0) { + for (;;) { c->xz.next_out = buffer; c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE; - lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN); + lzma_ret lzr = sym_lzma_code(&c->xz, size == 0 ? LZMA_FINISH : LZMA_RUN); if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) return -EBADMSG; - if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata); + if (c->xz.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - } - break; + if (lzr == LZMA_STREAM_END) { + if (c->xz.avail_in > 0) + return -EBADMSG; /* garbage data at the end */ + + c->finished = true; + return 0; + } + + if (c->xz.avail_in == 0 && c->xz.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif #if HAVE_LZ4 @@ -2014,7 +2051,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress const uint8_t *src = data; size_t src_remaining = size; - while (src_remaining > 0) { + c->finished = false; + + for (;;) { size_t produced = COMPRESS_PIPE_BUFFER_SIZE; size_t consumed = src_remaining; @@ -2022,8 +2061,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (sym_LZ4F_isError(rc)) return -EBADMSG; - if (consumed == 0 && produced == 0) - break; /* No progress possible with current input */ + if (consumed > src_remaining || + produced > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; src += consumed; src_remaining -= consumed; @@ -2033,9 +2073,24 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (r < 0) return r; } - } - break; + if (src_remaining == 0) { + if (rc == 0) { + /* The current stream has finished. */ + c->finished = true; + return 0; + } + + if (produced == COMPRESS_PIPE_BUFFER_SIZE) + /* Remaining data exist in the library buffer. We need to read them. */ + continue; + + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } } #endif @@ -2046,7 +2101,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress .size = size, }; - while (input.pos < input.size) { + c->finished = false; + + for (;;) { ZSTD_outBuffer output = { .dst = buffer, .size = COMPRESS_PIPE_BUFFER_SIZE, @@ -2056,14 +2113,33 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (sym_ZSTD_isError(res)) return -EBADMSG; + if (input.pos > input.size || + output.pos > output.size) + return -EIO; + if (output.pos > 0) { r = callback(output.dst, output.pos, userdata); if (r < 0) return r; } - } - break; + if (input.pos == input.size) { + if (res == 0) { + /* The current stream has finished. */ + c->finished = true; + return 0; + } + + if (output.pos == COMPRESS_PIPE_BUFFER_SIZE) + /* Remaining data exist in the library buffer. We need to read them. */ + continue; + + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } } #endif @@ -2075,7 +2151,14 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress c->gzip.next_in = (void*) data; c->gzip.avail_in = size; - while (c->gzip.avail_in > 0) { + for (;;) { + if (c->finished) { + if (sym_inflateReset(&c->gzip) != Z_OK) + return -EIO; + + c->finished = false; + } + c->gzip.next_out = buffer; c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE; @@ -2083,17 +2166,32 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (!IN_SET(zr, Z_OK, Z_STREAM_END)) return -EBADMSG; - if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata); + if (c->gzip.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - if (zr == Z_STREAM_END) - break; - } + if (zr == Z_STREAM_END) { + c->finished = true; - break; + if (c->gzip.avail_in > 0) + continue; /* New stream may be pending. */ + + return 0; + } + + if (c->gzip.avail_in == 0 && c->gzip.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif #if HAVE_BZIP2 @@ -2104,7 +2202,15 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress c->bzip2.next_in = (char*) data; c->bzip2.avail_in = size; - while (c->bzip2.avail_in > 0) { + for (;;) { + if (c->finished) { + sym_BZ2_bzDecompressEnd(&c->bzip2); + if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; + + c->finished = false; + } + c->bzip2.next_out = (char*) buffer; c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE; @@ -2112,24 +2218,37 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END)) return -EBADMSG; - if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata); + if (c->bzip2.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - if (bzr == BZ_STREAM_END) - break; - } + if (bzr == BZ_STREAM_END) { + c->finished = true; - break; + if (c->bzip2.avail_in > 0) + continue; /* New stream may be pending. */ + + return 0; + } + + if (c->bzip2.avail_in == 0 && c->bzip2.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif default: assert_not_reached(); } - - return 1; } int compressor_new(Compressor **ret, Compression type) { diff --git a/src/test/test-compress.c b/src/test/test-compress.c index 6ab482f718..5c419310b8 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -7,6 +7,7 @@ #include "alloc-util.h" #include "argv-util.h" #include "compress.h" +#include "copy.h" #include "fd-util.h" #include "io-util.h" #include "path-util.h" @@ -392,8 +393,7 @@ TEST(compress_decompress_stream) { log_debug("/* test faulty decompression */"); ASSERT_OK_ERRNO(lseek(dst, 1, SEEK_SET)); - r = decompress_stream(c, dst, dst2, st.st_size); - ASSERT_TRUE(IN_SET(r, 0, -EBADMSG)); + ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size), EBADMSG); ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET)); ASSERT_OK_ERRNO(lseek(dst2, 0, SEEK_SET)); @@ -520,6 +520,51 @@ TEST(decompress_stream_sparse) { compare_fd(src, decompressed); + log_debug("/* testing %s sparse decompression with multiple streams */", compression_to_string(c)); + { + _cleanup_close_ int multi = -EBADF, compressed_multi = -EBADF, decompressed_multi = -EBADF; + _cleanup_(unlink_tempfilep) char + pattern_multi[] = "/tmp/systemd-test.sparse-multi.XXXXXX", + pattern_compressed_multi[] = "/tmp/systemd-test.sparse-compressed_multi.XXXXXX", + pattern_decompressed_multi[] = "/tmp/systemd-test.sparse-decompressed_multi.XXXXXX"; + + ASSERT_OK(multi = mkostemp_safe(pattern_multi)); + ASSERT_OK(compressed_multi = mkostemp_safe(pattern_compressed_multi)); + ASSERT_OK(decompressed_multi = mkostemp_safe(pattern_decompressed_multi)); + + for (unsigned i = 0; i < 4; i++) { + ASSERT_OK(copy_bytes(src, multi, UINT64_MAX, COPY_SEEK0_SOURCE)); + ASSERT_OK(copy_bytes(compressed, compressed_multi, UINT64_MAX, COPY_SEEK0_SOURCE)); + } + + ASSERT_EQ(lseek(compressed_multi, 0, SEEK_SET), (off_t) 0); + ASSERT_OK(decompress_stream(c, compressed_multi, decompressed_multi, UINT64_MAX)); + + compare_fd(multi, decompressed_multi); + } + + log_debug("/* testing %s sparse decompression with garbage at the end */", compression_to_string(c)); + + off_t end = ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_END)); + ASSERT_OK(loop_write(compressed, "a", 1)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + + log_debug("/* testing %s sparse decompression with truncated data */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(compressed, end - 1)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + + log_debug("/* testing %s sparse decompression with zero length stream */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(compressed, 0)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + /* Test all-zeros input: entire output should be a hole */ log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c)); { @@ -619,9 +664,13 @@ TEST(compressor_decompressor_push_api) { ASSERT_EQ(compressor_type(decompressor), c); struct decompressor_test_data result = {}; - ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result)); - ASSERT_EQ(result.size, sizeof(text)); - ASSERT_EQ(memcmp(result.buf, text, sizeof(text)), 0); + for (unsigned i = 0; i < 4; i++) + ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result)); + ASSERT_OK(decompressor_push(decompressor, /* data= */ NULL, /* size= */ 0, test_decompressor_callback, &result)); + + ASSERT_EQ(result.size, sizeof(text) * 4); + for (unsigned i = 0; i < 4; i++) + ASSERT_EQ(memcmp(result.buf + sizeof(text) * i, text, sizeof(text)), 0); free(result.buf); decompressor = compressor_free(decompressor); From d8c0fa3cb8657816f5228e52ce751213bd2daae6 Mon Sep 17 00:00:00 2001 From: Yu Watanabe Date: Wed, 26 Aug 2026 05:12:28 +0900 Subject: [PATCH 9/9] import: finalize decompressor Otherwise, the result may be truncated, or trailing garbage may be silently accepted. --- src/import/pull-job.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 51714b1faa..a819911ca8 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -421,6 +421,11 @@ static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result if (j->state != PULL_JOB_RUNNING) return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.")); + /* Finalize decompressor. */ + r = decompressor_push(j->compress, /* data= */ NULL, /* size= */ 0, pull_job_write_uncompressed, j); + if (r < 0) + return pull_job_finish(j, r); + uint64_t cl = pull_job_content_length_effective(j); if (cl != UINT64_MAX && cl != j->written_compressed)