diff --git a/src/basic/compress.c b/src/basic/compress.c index 8cbf22005d..f3e57dd284 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -133,6 +133,7 @@ static DLSYM_PROTOTYPE(deflateEnd) = NULL; static DLSYM_PROTOTYPE(inflateInit2_) = NULL; static DLSYM_PROTOTYPE(inflate) = NULL; static DLSYM_PROTOTYPE(inflateEnd) = NULL; +static DLSYM_PROTOTYPE(inflateReset) = NULL; static inline void deflateEnd_wrapper(z_stream *s) { sym_deflateEnd(s); @@ -164,6 +165,7 @@ static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) { struct Compressor { Compression type; bool encoding; + bool finished; union { #if HAVE_XZ lzma_stream xz; @@ -396,7 +398,8 @@ int dlopen_zlib(int log_level) { DLSYM_ARG(deflateEnd), DLSYM_ARG(inflateInit2_), DLSYM_ARG(inflate), - DLSYM_ARG(inflateEnd)); + DLSYM_ARG(inflateEnd), + DLSYM_ARG(inflateReset)); #else return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP), "zlib support is not compiled in."); @@ -482,9 +485,9 @@ static int compress_blob_xz( return r; if (level >= 0) { - r = sym_lzma_lzma_preset(&opt, (uint32_t) level); - if (r < 0) - return r; + /* This returns lzma_bool, and true means failure, i.e. the specified level is not supported. */ + if (sym_lzma_lzma_preset(&opt, (uint32_t) level)) + return -EINVAL; } /* Returns < 0 if we couldn't compress the data or the @@ -493,8 +496,8 @@ static int compress_blob_xz( if (src_size < 80) return -ENOBUFS; - ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL, - src, src_size, dst, &out_pos, dst_alloc_size); + ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, /* allocator= */ NULL, + src, src_size, dst, &out_pos, dst_alloc_size); if (ret != LZMA_OK) return -ENOBUFS; @@ -1040,9 +1043,8 @@ static int decompress_blob_bzip2( _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; - r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) - return -ENOMEM; + if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX); if (!greedy_realloc(dst, space, 1)) @@ -1477,9 +1479,8 @@ static int decompress_startswith_bzip2( _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; - r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) - return -EBADMSG; + if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1))) return -ENOMEM; @@ -1634,21 +1635,34 @@ int compress_stream( return 0; } -/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on - * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */ static int should_sparse(int fd) { struct stat st; assert(fd >= 0); + /* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on regular + * files, without O_APPEND (which ignores the lseek position, collapsing any holes we try to create), + * and only when starting from a position at or beyond the current end of file (otherwise the skipped + * ranges would overlap pre-existing data instead of being genuinely unwritten). */ + if (fstat(fd, &st) < 0) return -errno; + if (!S_ISREG(st.st_mode)) + return false; + int flags = fcntl(fd, F_GETFL); if (flags < 0) return -errno; - return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND); + if (FLAGS_SET(flags, O_APPEND)) + return false; + + off_t pos = lseek(fd, 0, SEEK_CUR); + if (pos < 0) + return -errno; + + return pos >= st.st_size; } /* After sparse decompression, set the file size to the current position to account for @@ -1702,10 +1716,6 @@ static int decompress_stream_write_callback(const void *data, size_t size, void } static int decompressor_new(Decompressor **ret, Compression type) { -#if HAVE_ZLIB || HAVE_BZIP2 - int r; -#endif - assert(ret); _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1); @@ -1745,16 +1755,14 @@ static int decompressor_new(Decompressor **ret, Compression type) { #if HAVE_ZLIB case COMPRESSION_GZIP: - r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)); - if (r != Z_OK) + if (sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)) != Z_OK) return -EIO; break; #endif #if HAVE_BZIP2 case COMPRESSION_BZIP2: - r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0); - if (r != BZ_OK) + if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) return -EIO; break; #endif @@ -1809,14 +1817,15 @@ int decompress_stream( n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE); if (n < 0) return -errno; - if (n == 0) - break; total_in += n; r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata); if (r < 0) return r; + + if (n == 0) + break; } if (total_in == 0) @@ -1953,15 +1962,20 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress int r; assert(c); + assert(data || size == 0); assert(callback); + /* This feeds `size` bytes of compressed data into `c`. Passing size == 0 has special meaning: it + * signals that no further input will be provided, and the function will attempt to finalize the + * stream, failing with -EBADMSG if the compressed data was truncated or has trailing garbage after + * the logical end of stream. */ + if (c->encoding) return -EINVAL; - if (size == 0) - return 1; - - assert(data); + if (c->finished && size == 0) + /* The previous stream was correctly finished, and it was the last one. */ + return 0; #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 _cleanup_free_ uint8_t *buffer = NULL; @@ -1975,33 +1989,61 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress switch (c->type) { case COMPRESSION_NONE: + if (c->finished) + return -EBADMSG; + + if (size == 0) { + c->finished = true; + return 0; + } + r = callback(data, size, userdata); if (r < 0) return r; - break; + return 0; #if HAVE_XZ case COMPRESSION_XZ: + if (c->finished) + return -EBADMSG; + c->xz.next_in = data; c->xz.avail_in = size; - while (c->xz.avail_in > 0) { + for (;;) { c->xz.next_out = buffer; c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE; - lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN); + lzma_ret lzr = sym_lzma_code(&c->xz, size == 0 ? LZMA_FINISH : LZMA_RUN); if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) return -EBADMSG; - if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata); + if (c->xz.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - } - break; + if (lzr == LZMA_STREAM_END) { + if (c->xz.avail_in > 0) + return -EBADMSG; /* garbage data at the end */ + + c->finished = true; + return 0; + } + + if (c->xz.avail_in == 0 && c->xz.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif #if HAVE_LZ4 @@ -2009,7 +2051,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress const uint8_t *src = data; size_t src_remaining = size; - while (src_remaining > 0) { + c->finished = false; + + for (;;) { size_t produced = COMPRESS_PIPE_BUFFER_SIZE; size_t consumed = src_remaining; @@ -2017,8 +2061,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (sym_LZ4F_isError(rc)) return -EBADMSG; - if (consumed == 0 && produced == 0) - break; /* No progress possible with current input */ + if (consumed > src_remaining || + produced > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; src += consumed; src_remaining -= consumed; @@ -2028,9 +2073,24 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (r < 0) return r; } - } - break; + if (src_remaining == 0) { + if (rc == 0) { + /* The current stream has finished. */ + c->finished = true; + return 0; + } + + if (produced == COMPRESS_PIPE_BUFFER_SIZE) + /* Remaining data exist in the library buffer. We need to read them. */ + continue; + + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } } #endif @@ -2041,7 +2101,9 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress .size = size, }; - while (input.pos < input.size) { + c->finished = false; + + for (;;) { ZSTD_outBuffer output = { .dst = buffer, .size = COMPRESS_PIPE_BUFFER_SIZE, @@ -2051,14 +2113,33 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (sym_ZSTD_isError(res)) return -EBADMSG; + if (input.pos > input.size || + output.pos > output.size) + return -EIO; + if (output.pos > 0) { r = callback(output.dst, output.pos, userdata); if (r < 0) return r; } - } - break; + if (input.pos == input.size) { + if (res == 0) { + /* The current stream has finished. */ + c->finished = true; + return 0; + } + + if (output.pos == COMPRESS_PIPE_BUFFER_SIZE) + /* Remaining data exist in the library buffer. We need to read them. */ + continue; + + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } } #endif @@ -2070,7 +2151,14 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress c->gzip.next_in = (void*) data; c->gzip.avail_in = size; - while (c->gzip.avail_in > 0) { + for (;;) { + if (c->finished) { + if (sym_inflateReset(&c->gzip) != Z_OK) + return -EIO; + + c->finished = false; + } + c->gzip.next_out = buffer; c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE; @@ -2078,17 +2166,32 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (!IN_SET(zr, Z_OK, Z_STREAM_END)) return -EBADMSG; - if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata); + if (c->gzip.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - if (zr == Z_STREAM_END) - break; - } + if (zr == Z_STREAM_END) { + c->finished = true; - break; + if (c->gzip.avail_in > 0) + continue; /* New stream may be pending. */ + + return 0; + } + + if (c->gzip.avail_in == 0 && c->gzip.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif #if HAVE_BZIP2 @@ -2099,7 +2202,15 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress c->bzip2.next_in = (char*) data; c->bzip2.avail_in = size; - while (c->bzip2.avail_in > 0) { + for (;;) { + if (c->finished) { + sym_BZ2_bzDecompressEnd(&c->bzip2); + if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK) + return -EIO; + + c->finished = false; + } + c->bzip2.next_out = (char*) buffer; c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE; @@ -2107,24 +2218,37 @@ int decompressor_push(Decompressor *c, const void *data, size_t size, Decompress if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END)) return -EBADMSG; - if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { - r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata); + if (c->bzip2.avail_out > COMPRESS_PIPE_BUFFER_SIZE) + return -EIO; + + size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out; + if (produced > 0) { + r = callback(buffer, produced, userdata); if (r < 0) return r; } - if (bzr == BZ_STREAM_END) - break; - } + if (bzr == BZ_STREAM_END) { + c->finished = true; - break; + if (c->bzip2.avail_in > 0) + continue; /* New stream may be pending. */ + + return 0; + } + + if (c->bzip2.avail_in == 0 && c->bzip2.avail_out > 0) { + if (size == 0) + return -EBADMSG; /* truncated */ + + return 0; /* No progress possible with the current input. */ + } + } #endif default: assert_not_reached(); } - - return 1; } int compressor_new(Compressor **ret, Compression type) { diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 51714b1faa..a819911ca8 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -421,6 +421,11 @@ static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result if (j->state != PULL_JOB_RUNNING) return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.")); + /* Finalize decompressor. */ + r = decompressor_push(j->compress, /* data= */ NULL, /* size= */ 0, pull_job_write_uncompressed, j); + if (r < 0) + return pull_job_finish(j, r); + uint64_t cl = pull_job_content_length_effective(j); if (cl != UINT64_MAX && cl != j->written_compressed) diff --git a/src/test/test-compress.c b/src/test/test-compress.c index bfffcc4804..5c419310b8 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -7,6 +7,7 @@ #include "alloc-util.h" #include "argv-util.h" #include "compress.h" +#include "copy.h" #include "fd-util.h" #include "io-util.h" #include "path-util.h" @@ -392,8 +393,7 @@ TEST(compress_decompress_stream) { log_debug("/* test faulty decompression */"); ASSERT_OK_ERRNO(lseek(dst, 1, SEEK_SET)); - r = decompress_stream(c, dst, dst2, st.st_size); - ASSERT_TRUE(IN_SET(r, 0, -EBADMSG)); + ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size), EBADMSG); ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET)); ASSERT_OK_ERRNO(lseek(dst2, 0, SEEK_SET)); @@ -417,6 +417,29 @@ static int test_decompressor_callback(const void *p, size_t size, void *userdata return 0; } +static void compare_fd(int fda, int fdb) { + struct stat sta, stb; + + /* Verify apparent size matches */ + ASSERT_OK_ERRNO(fstat(fda, &sta)); + ASSERT_OK_ERRNO(fstat(fdb, &stb)); + ASSERT_EQ(sta.st_size, stb.st_size); + + /* Verify content matches by comparing bytes */ + ASSERT_OK_EQ_ERRNO(lseek(fda, 0, SEEK_SET), (off_t) 0); + ASSERT_OK_EQ_ERRNO(lseek(fdb, 0, SEEK_SET), (off_t) 0); + + for (off_t offset = 0; offset < sta.st_size;) { + uint8_t bufa[4096], bufb[4096]; + size_t to_read = MIN((size_t) (sta.st_size - offset), sizeof(bufa)); + + ASSERT_OK(loop_read_exact(fda, bufa, to_read, /* do_poll= */ false)); + ASSERT_OK(loop_read_exact(fdb, bufb, to_read, /* do_poll= */ false)); + ASSERT_EQ(memcmp(bufa, bufb, to_read), 0); + offset += to_read; + } +} + TEST(decompress_stream_sparse) { for (Compression c = 0; c < _COMPRESSION_MAX; c++) { if (c == COMPRESSION_NONE || !compression_supported(c)) @@ -460,24 +483,9 @@ TEST(decompress_stream_sparse) { ASSERT_EQ(lseek(compressed, 0, SEEK_SET), (off_t) 0); ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size)); - /* Verify apparent size matches */ + compare_fd(src, decompressed); + ASSERT_OK_ERRNO(fstat(decompressed, &st_decompressed)); - ASSERT_EQ(st_decompressed.st_size, st_src.st_size); - - /* Verify content matches by comparing bytes */ - ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0); - ASSERT_EQ(lseek(decompressed, 0, SEEK_SET), (off_t) 0); - - for (off_t offset = 0; offset < st_src.st_size;) { - uint8_t buf_src[4096], buf_dst[4096]; - size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src)); - - ASSERT_EQ(loop_read(src, buf_src, to_read, true), (ssize_t) to_read); - ASSERT_EQ(loop_read(decompressed, buf_dst, to_read, true), (ssize_t) to_read); - ASSERT_EQ(memcmp(buf_src, buf_dst, to_read), 0); - offset += to_read; - } - /* Verify the decompressed file is actually sparse (uses less disk than apparent size). * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be * noticeably less than the apparent size if sparse writes worked. @@ -491,6 +499,72 @@ TEST(decompress_stream_sparse) { else log_debug("Filesystem does not support holes, skipping sparsity check"); + /* Check if the decompressor clears previous data. + * The original data is 4K data, 64K zeros, 4K data, 64K zeros. + * Now, fill the file with 4K zeros, 64K data, 4K zeros, 64K data. */ + log_debug("/* testing %s sparse decompression to an existing file */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(decompressed, 0)); + ASSERT_OK_ERRNO(ftruncate(decompressed, 4096)); + ASSERT_OK_ERRNO(lseek(decompressed, 4096, SEEK_SET)); + for (unsigned i = 0; i < 16; i++) + ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block))); + ASSERT_OK_ERRNO(ftruncate(decompressed, 4096 + 16 * sizeof(data_block) + 4096)); + ASSERT_OK_ERRNO(lseek(decompressed, 4096 + 16 * sizeof(data_block) + 4096, SEEK_SET)); + for (unsigned i = 0; i < 16; i++) + ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block))); + + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size)); + + compare_fd(src, decompressed); + + log_debug("/* testing %s sparse decompression with multiple streams */", compression_to_string(c)); + { + _cleanup_close_ int multi = -EBADF, compressed_multi = -EBADF, decompressed_multi = -EBADF; + _cleanup_(unlink_tempfilep) char + pattern_multi[] = "/tmp/systemd-test.sparse-multi.XXXXXX", + pattern_compressed_multi[] = "/tmp/systemd-test.sparse-compressed_multi.XXXXXX", + pattern_decompressed_multi[] = "/tmp/systemd-test.sparse-decompressed_multi.XXXXXX"; + + ASSERT_OK(multi = mkostemp_safe(pattern_multi)); + ASSERT_OK(compressed_multi = mkostemp_safe(pattern_compressed_multi)); + ASSERT_OK(decompressed_multi = mkostemp_safe(pattern_decompressed_multi)); + + for (unsigned i = 0; i < 4; i++) { + ASSERT_OK(copy_bytes(src, multi, UINT64_MAX, COPY_SEEK0_SOURCE)); + ASSERT_OK(copy_bytes(compressed, compressed_multi, UINT64_MAX, COPY_SEEK0_SOURCE)); + } + + ASSERT_EQ(lseek(compressed_multi, 0, SEEK_SET), (off_t) 0); + ASSERT_OK(decompress_stream(c, compressed_multi, decompressed_multi, UINT64_MAX)); + + compare_fd(multi, decompressed_multi); + } + + log_debug("/* testing %s sparse decompression with garbage at the end */", compression_to_string(c)); + + off_t end = ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_END)); + ASSERT_OK(loop_write(compressed, "a", 1)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + + log_debug("/* testing %s sparse decompression with truncated data */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(compressed, end - 1)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + + log_debug("/* testing %s sparse decompression with zero length stream */", compression_to_string(c)); + + ASSERT_OK_ERRNO(ftruncate(compressed, 0)); + ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG); + /* Test all-zeros input: entire output should be a hole */ log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c)); { @@ -500,23 +574,23 @@ TEST(decompress_stream_sparse) { zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX", zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX"; struct stat zst; - uint64_t zsize; - uint8_t zeros[65536] = {}; + uint64_t zsize, zsize_expected = 65536; ASSERT_OK(zsrc = mkostemp_safe(zp_src)); - ASSERT_OK(loop_write(zsrc, zeros, sizeof(zeros))); + ASSERT_OK_ERRNO(ftruncate(zsrc, zsize_expected)); ASSERT_EQ(lseek(zsrc, 0, SEEK_SET), (off_t) 0); ASSERT_OK(zcompressed = mkostemp_safe(zp_compressed)); ASSERT_OK(compress_stream(c, zsrc, zcompressed, -1, &zsize)); - ASSERT_EQ(zsize, (uint64_t) sizeof(zeros)); + ASSERT_EQ(zsize, zsize_expected); ASSERT_OK(zdecompressed = mkostemp_safe(zp_decompressed)); ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0); - ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, sizeof(zeros))); + ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, zsize_expected)); + + compare_fd(zsrc, zdecompressed); ASSERT_OK_ERRNO(fstat(zdecompressed, &zst)); - ASSERT_EQ(zst.st_size, (off_t) sizeof(zeros)); /* All zeros — disk usage should be minimal */ log_debug("%s all-zeros sparse: apparent=%jd disk=%jd", compression_to_string(c), (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512); @@ -534,26 +608,24 @@ TEST(decompress_stream_sparse) { dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX", dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; - struct stat dst; uint64_t dsize; - uint8_t zeros[65536] = {}; /* 64K zeros followed by 4K random data */ ASSERT_OK(dsrc = mkostemp_safe(dp_src)); - ASSERT_OK(loop_write(dsrc, zeros, sizeof(zeros))); + ASSERT_OK_ERRNO(ftruncate(dsrc, 65536)); + ASSERT_OK_ERRNO(lseek(dsrc, 65536, SEEK_SET)); ASSERT_OK(loop_write(dsrc, data_block, sizeof(data_block))); ASSERT_EQ(lseek(dsrc, 0, SEEK_SET), (off_t) 0); ASSERT_OK(dcompressed = mkostemp_safe(dp_compressed)); ASSERT_OK(compress_stream(c, dsrc, dcompressed, -1, &dsize)); - ASSERT_EQ(dsize, (uint64_t)(sizeof(zeros) + sizeof(data_block))); + ASSERT_EQ(dsize, (uint64_t) (65536 + sizeof(data_block))); ASSERT_OK(ddecompressed = mkostemp_safe(dp_decompressed)); ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0); ASSERT_OK_ZERO(decompress_stream(c, dcompressed, ddecompressed, dsize)); - ASSERT_OK_ERRNO(fstat(ddecompressed, &dst)); - ASSERT_EQ(dst.st_size, (off_t)(sizeof(zeros) + sizeof(data_block))); + compare_fd(dsrc, ddecompressed); } } } @@ -592,9 +664,13 @@ TEST(compressor_decompressor_push_api) { ASSERT_EQ(compressor_type(decompressor), c); struct decompressor_test_data result = {}; - ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result)); - ASSERT_EQ(result.size, sizeof(text)); - ASSERT_EQ(memcmp(result.buf, text, sizeof(text)), 0); + for (unsigned i = 0; i < 4; i++) + ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result)); + ASSERT_OK(decompressor_push(decompressor, /* data= */ NULL, /* size= */ 0, test_decompressor_callback, &result)); + + ASSERT_EQ(result.size, sizeof(text) * 4); + for (unsigned i = 0; i < 4; i++) + ASSERT_EQ(memcmp(result.buf + sizeof(text) * i, text, sizeof(text)), 0); free(result.buf); decompressor = compressor_free(decompressor);