SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7ea7833b86d243a2ff5cb8b63cb469be25579bbe
systemd/src/basic/compress.c
T

2608 lines
80 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#if HAVE_XZ
#ifndef SYSTEMD_CFLAGS_MARKER_LIBXZ
# error "missing libxz_cflags in meson dependency."
#endif
#include <lzma.h>
#endif
#if HAVE_LZ4
#ifndef SYSTEMD_CFLAGS_MARKER_LIBLZ4
# error "missing liblz4_cflags in meson dependency."
#endif
#include <lz4.h>
#include <lz4frame.h>
#include <lz4hc.h>
#endif
#if HAVE_ZSTD
#ifndef SYSTEMD_CFLAGS_MARKER_LIBZSTD
# error "missing libzstd_cflags in meson dependency."
#endif
#include <zstd.h>
#include <zstd_errors.h>
#endif
#if HAVE_ZLIB
#ifndef SYSTEMD_CFLAGS_MARKER_LIBZ
# error "missing libz_cflags in meson dependency."
#endif
#include <zlib.h>
#endif
#if HAVE_BZIP2
#ifndef SYSTEMD_CFLAGS_MARKER_LIBBZIP2
# error "missing libbzip2_cflags in meson dependency."
#endif
#include <bzlib.h>
#endif
#include "alloc-util.h"
#include "bitfield.h"
#include "compress.h"
#include "dlfcn-util.h"
#include "io-util.h"
#include "log.h"
#include "string-table.h"
#include "unaligned.h"
#if HAVE_XZ
static DLSYM_PROTOTYPE(lzma_code) = NULL;
static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
static DLSYM_PROTOTYPE(lzma_end) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;
/* We can’t just do _cleanup_(sym_lzma_end) because a compiler bug makes
* this fail with:
* ../src/basic/compress.c: In function ‘decompress_blob_xz’:
* ../src/basic/compress.c:304:9: error: cleanup argument not a function
* 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
* | ^~~~~~~~~
*/
static inline void lzma_end_wrapper(lzma_stream *ls) {
sym_lzma_end(ls);
}
#endif
#if HAVE_LZ4
static DLSYM_PROTOTYPE(LZ4F_compressBegin) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressBound) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressEnd) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressUpdate) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_decompress) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_isError) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
static DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
static DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
static DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
static const LZ4F_preferences_t lz4_preferences = {
.frameInfo.blockSizeID = 5,
};
#endif
#if HAVE_ZSTD
static DLSYM_PROTOTYPE(ZSTD_CCtx_setParameter) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compress) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compressStream2) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_decompressStream) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorCode) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorName) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_isError) = NULL;
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL);
static int zstd_ret_to_errno(size_t ret) {
switch (sym_ZSTD_getErrorCode(ret)) {
case ZSTD_error_dstSize_tooSmall:
return -ENOBUFS;
case ZSTD_error_memory_allocation:
return -ENOMEM;
default:
return -EBADMSG;
}
}
#endif
#if HAVE_ZLIB
static DLSYM_PROTOTYPE(deflateInit2_) = NULL;
static DLSYM_PROTOTYPE(deflate) = NULL;
static DLSYM_PROTOTYPE(deflateEnd) = NULL;
static DLSYM_PROTOTYPE(inflateInit2_) = NULL;
static DLSYM_PROTOTYPE(inflate) = NULL;
static DLSYM_PROTOTYPE(inflateEnd) = NULL;
static inline void deflateEnd_wrapper(z_stream *s) {
sym_deflateEnd(s);
}
static inline void inflateEnd_wrapper(z_stream *s) {
sym_inflateEnd(s);
}
#endif
#if HAVE_BZIP2
static DLSYM_PROTOTYPE(BZ2_bzCompressInit) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzCompress) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzCompressEnd) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompressInit) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompress) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompressEnd) = NULL;
static inline void BZ2_bzCompressEnd_wrapper(bz_stream *s) {
sym_BZ2_bzCompressEnd(s);
}
static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) {
sym_BZ2_bzDecompressEnd(s);
}
#endif
/* Opaque Compressor/Decompressor struct definition */
struct Compressor {
Compression type;
bool encoding;
union {
#if HAVE_XZ
lzma_stream xz;
#endif
#if HAVE_LZ4
struct {
LZ4F_compressionContext_t c_lz4;
void *lz4_header; /* stashed frame header from LZ4F_compressBegin */
size_t lz4_header_size;
};
LZ4F_decompressionContext_t d_lz4;
#endif
#if HAVE_ZSTD
ZSTD_CCtx *c_zstd;
ZSTD_DCtx *d_zstd;
#endif
#if HAVE_ZLIB
z_stream gzip;
#endif
#if HAVE_BZIP2
bz_stream bzip2;
#endif
};
};
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
/* zlib windowBits value for gzip format: MAX_WBITS (15) + 16 to enable gzip header detection/generation */
#define ZLIB_WBITS_GZIP (15 + 16)
static const char* const compression_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "uncompressed", /* backwards compatibility with importd */
[COMPRESSION_XZ] = "xz",
[COMPRESSION_LZ4] = "lz4",
[COMPRESSION_ZSTD] = "zstd",
[COMPRESSION_GZIP] = "gzip",
[COMPRESSION_BZIP2] = "bzip2",
};
static const char* const compression_uppercase_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "NONE", /* backwards compatibility with SYSTEMD_JOURNAL_COMPRESS=NONE */
[COMPRESSION_XZ] = "XZ",
[COMPRESSION_LZ4] = "LZ4",
[COMPRESSION_ZSTD] = "ZSTD",
[COMPRESSION_GZIP] = "GZIP",
[COMPRESSION_BZIP2] = "BZIP2",
};
static const char* const compression_extension_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "",
[COMPRESSION_XZ] = ".xz",
[COMPRESSION_LZ4] = ".lz4",
[COMPRESSION_ZSTD] = ".zst",
[COMPRESSION_GZIP] = ".gz",
[COMPRESSION_BZIP2] = ".bz2",
};
DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
DEFINE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
DEFINE_STRING_TABLE_LOOKUP(compression_extension, Compression);
Compression compression_from_string_harder(const char *s) {
Compression c;
assert(s);
c = compression_from_string(s);
if (c >= 0)
return c;
return compression_uppercase_from_string(s);
}
Compression compression_from_filename(const char *filename) {
Compression c;
const char *e;
assert(filename);
e = strrchr(filename, '.');
if (!e)
return COMPRESSION_NONE;
c = compression_extension_from_string(e);
if (c < 0)
return COMPRESSION_NONE;
return c;
}
bool compression_supported_journal(Compression c) {
static const unsigned supported =
(1U << COMPRESSION_NONE) |
(1U << COMPRESSION_XZ) * HAVE_XZ |
(1U << COMPRESSION_LZ4) * HAVE_LZ4 |
(1U << COMPRESSION_ZSTD) * HAVE_ZSTD;
assert(c >= 0);
assert(c < _COMPRESSION_MAX);
return BIT_SET(supported, c);
}
bool compression_supported(Compression c) {
static const unsigned supported =
(1U << COMPRESSION_NONE) |
(1U << COMPRESSION_XZ) * HAVE_XZ |
(1U << COMPRESSION_LZ4) * HAVE_LZ4 |
(1U << COMPRESSION_ZSTD) * HAVE_ZSTD |
(1U << COMPRESSION_GZIP) * HAVE_ZLIB |
(1U << COMPRESSION_BZIP2) * HAVE_BZIP2;
assert(c >= 0);
assert(c < _COMPRESSION_MAX);
return BIT_SET(supported, c);
}
Compression compression_detect_from_magic(const uint8_t data[static COMPRESSION_MAGIC_BYTES_MAX]) {
/* Magic signatures per RFC 1952 (gzip), tukaani.org/xz/xz-file-format.txt (xz),
* RFC 8878 (zstd), lz4/doc/lz4_Frame_format.md (lz4), and the bzip2 file format.
* Make sure to update COMPRESSION_MAGIC_BYTES_MAX if needed when adding a new magic. */
if (memcmp(data, (const uint8_t[]) { 0x1f, 0x8b }, 2) == 0)
return COMPRESSION_GZIP;
if (memcmp(data, (const uint8_t[]) { 0xfd, '7', 'z', 'X', 'Z', 0x00 }, 6) == 0)
return COMPRESSION_XZ;
if (memcmp(data, (const uint8_t[]) { 0x28, 0xb5, 0x2f, 0xfd }, 4) == 0)
return COMPRESSION_ZSTD;
if (memcmp(data, (const uint8_t[]) { 0x04, 0x22, 0x4d, 0x18 }, 4) == 0)
return COMPRESSION_LZ4;
if (memcmp(data, (const uint8_t[]) { 'B', 'Z', 'h' }, 3) == 0)
return COMPRESSION_BZIP2;
return _COMPRESSION_INVALID;
}
int dlopen_xz(int log_level) {
#if HAVE_XZ
static void *lzma_dl = NULL;
LIBLZMA_NOTE(suggested);
return dlopen_many_sym_or_warn(
&lzma_dl,
"liblzma.so.5", log_level,
DLSYM_ARG(lzma_code),
DLSYM_ARG(lzma_easy_encoder),
DLSYM_ARG(lzma_end),
DLSYM_ARG(lzma_stream_buffer_encode),
DLSYM_ARG(lzma_lzma_preset),
DLSYM_ARG(lzma_stream_decoder));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"lzma support is not compiled in.");
#endif
}
int dlopen_lz4(int log_level) {
#if HAVE_LZ4
static void *lz4_dl = NULL;
LIBLZ4_NOTE(suggested);
return dlopen_many_sym_or_warn(
&lz4_dl,
"liblz4.so.1", log_level,
DLSYM_ARG(LZ4F_compressBegin),
DLSYM_ARG(LZ4F_compressBound),
DLSYM_ARG(LZ4F_compressEnd),
DLSYM_ARG(LZ4F_compressUpdate),
DLSYM_ARG(LZ4F_createCompressionContext),
DLSYM_ARG(LZ4F_createDecompressionContext),
DLSYM_ARG(LZ4F_decompress),
DLSYM_ARG(LZ4F_freeCompressionContext),
DLSYM_ARG(LZ4F_freeDecompressionContext),
DLSYM_ARG(LZ4F_isError),
DLSYM_ARG(LZ4_compress_default),
DLSYM_ARG(LZ4_compress_HC),
DLSYM_ARG(LZ4_decompress_safe),
DLSYM_ARG(LZ4_decompress_safe_partial),
DLSYM_ARG(LZ4_versionNumber));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"lz4 support is not compiled in.");
#endif
}
int dlopen_zstd(int log_level) {
#if HAVE_ZSTD
static void *zstd_dl = NULL;
LIBZSTD_NOTE(suggested);
return dlopen_many_sym_or_warn(
&zstd_dl,
"libzstd.so.1", log_level,
DLSYM_ARG(ZSTD_getErrorCode),
DLSYM_ARG(ZSTD_compress),
DLSYM_ARG(ZSTD_getFrameContentSize),
DLSYM_ARG(ZSTD_decompressStream),
DLSYM_ARG(ZSTD_getErrorName),
DLSYM_ARG(ZSTD_DStreamOutSize),
DLSYM_ARG(ZSTD_CStreamInSize),
DLSYM_ARG(ZSTD_CStreamOutSize),
DLSYM_ARG(ZSTD_CCtx_setParameter),
DLSYM_ARG(ZSTD_compressStream2),
DLSYM_ARG(ZSTD_DStreamInSize),
DLSYM_ARG(ZSTD_freeCCtx),
DLSYM_ARG(ZSTD_freeDCtx),
DLSYM_ARG(ZSTD_isError),
DLSYM_ARG(ZSTD_createDCtx),
DLSYM_ARG(ZSTD_createCCtx));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"zstd support is not compiled in.");
#endif
}
int dlopen_zlib(int log_level) {
#if HAVE_ZLIB
static void *zlib_dl = NULL;
LIBZ_NOTE(suggested);
return dlopen_many_sym_or_warn(
&zlib_dl,
"libz.so.1", log_level,
DLSYM_ARG(deflateInit2_),
DLSYM_ARG(deflate),
DLSYM_ARG(deflateEnd),
DLSYM_ARG(inflateInit2_),
DLSYM_ARG(inflate),
DLSYM_ARG(inflateEnd));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"zlib support is not compiled in.");
#endif
}
int dlopen_bzip2(int log_level) {
#if HAVE_BZIP2
static void *bzip2_dl = NULL;
LIBBZ2_NOTE(suggested);
return dlopen_many_sym_or_warn(
&bzip2_dl,
"libbz2.so.1", log_level,
DLSYM_ARG(BZ2_bzCompressInit),
DLSYM_ARG(BZ2_bzCompress),
DLSYM_ARG(BZ2_bzCompressEnd),
DLSYM_ARG(BZ2_bzDecompressInit),
DLSYM_ARG(BZ2_bzDecompress),
DLSYM_ARG(BZ2_bzDecompressEnd));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"bzip2 support is not compiled in.");
#endif
}
int dlopen_compress_journal(Compression c, int log_level) {
switch (c) {
case COMPRESSION_NONE:
return 0;
case COMPRESSION_XZ:
return dlopen_xz(log_level);
case COMPRESSION_LZ4:
return dlopen_lz4(log_level);
case COMPRESSION_ZSTD:
return dlopen_zstd(log_level);
default:
return -EOPNOTSUPP;
}
}
int dlopen_compress(Compression c, int log_level) {
switch (c) {
case COMPRESSION_GZIP:
return dlopen_zlib(log_level);
case COMPRESSION_BZIP2:
return dlopen_bzip2(log_level);
default:
return dlopen_compress_journal(c, log_level);
}
}
static int compress_blob_xz(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_XZ
lzma_options_lzma opt = {
1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
};
lzma_filter filters[] = {
{ LZMA_FILTER_LZMA2, &opt },
{ LZMA_VLI_UNKNOWN, NULL }
};
lzma_ret ret;
size_t out_pos = 0;
int r;
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
if (level >= 0) {
r = sym_lzma_lzma_preset(&opt, (uint32_t) level);
if (r < 0)
return r;
}
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
if (src_size < 80)
return -ENOBUFS;
ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL,
src, src_size, dst, &out_pos, dst_alloc_size);
if (ret != LZMA_OK)
return -ENOBUFS;
*dst_size = out_pos;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_lz4(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_LZ4
int r;
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
if (src_size < 9)
return -ENOBUFS;
if (src_size > INT_MAX)
return -EFBIG;
if (dst_alloc_size > INT_MAX)
dst_alloc_size = INT_MAX;
if (level <= 0)
r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
else
r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level);
if (r <= 0)
return -ENOBUFS;
unaligned_write_le64(dst, src_size);
*dst_size = r + 8;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_zstd(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_ZSTD
size_t k;
int r;
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, level < 0 ? 0 : level);
if (sym_ZSTD_isError(k))
return zstd_ret_to_errno(k);
*dst_size = k;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_gzip(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
if (dst_alloc_size > UINT_MAX)
dst_alloc_size = UINT_MAX;
_cleanup_(deflateEnd_wrapper) z_stream s = {};
r = sym_deflateInit2_(&s, level < 0 ? Z_DEFAULT_COMPRESSION : level,
/* method= */ Z_DEFLATED,
/* windowBits= */ ZLIB_WBITS_GZIP,
/* memLevel= */ 8,
/* strategy= */ Z_DEFAULT_STRATEGY,
ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -ENOMEM;
s.next_in = (void*) src;
s.avail_in = src_size;
s.next_out = dst;
s.avail_out = dst_alloc_size;
r = sym_deflate(&s, Z_FINISH);
if (r != Z_STREAM_END)
return -ENOBUFS;
*dst_size = dst_alloc_size - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_bzip2(
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_BZIP2
int r;
r = dlopen_bzip2(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
if (dst_alloc_size > UINT_MAX)
dst_alloc_size = UINT_MAX;
_cleanup_(BZ2_bzCompressEnd_wrapper) bz_stream s = {};
r = sym_BZ2_bzCompressInit(&s, level < 0 ? 9 : level, /* verbosity= */ 0, /* workFactor= */ 0);
if (r != BZ_OK)
return -ENOMEM;
s.next_in = (char*) src;
s.avail_in = src_size;
s.next_out = (char*) dst;
s.avail_out = dst_alloc_size;
r = sym_BZ2_bzCompress(&s, BZ_FINISH);
if (r != BZ_STREAM_END)
return -ENOBUFS;
*dst_size = dst_alloc_size - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
int compress_blob_journal(
Compression compression,
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
switch (compression) {
case COMPRESSION_XZ:
return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_LZ4:
return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_ZSTD:
return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
default:
return -EOPNOTSUPP;
}
}
int compress_blob(
Compression compression,
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
switch (compression) {
case COMPRESSION_GZIP:
return compress_blob_gzip(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_BZIP2:
return compress_blob_bzip2(src, src_size, dst, dst_alloc_size, dst_size, level);
default:
return compress_blob_journal(compression, src, src_size, dst, dst_alloc_size, dst_size, level);
}
}
static int decompress_blob_xz(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_XZ
int r;
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
if (ret != LZMA_OK)
return -ENOMEM;
size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.next_in = src;
s.avail_in = src_size;
s.next_out = *dst;
s.avail_out = space;
for (;;) {
size_t used;
ret = sym_lzma_code(&s, LZMA_FINISH);
if (ret == LZMA_STREAM_END)
break;
if (ret != LZMA_OK)
return -ENOMEM;
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
break;
if (dst_max > 0 && space == dst_max)
return -ENOBUFS;
used = space - s.avail_out;
/* Silence static analyzers, space is bounded by allocation size */
assert(space <= SIZE_MAX / 2);
space = MIN(2 * space, dst_max ?: SIZE_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.avail_out = space - used;
s.next_out = *(uint8_t**)dst + used;
}
*dst_size = space - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_blob_lz4(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_LZ4
char* out;
int r, size; /* LZ4 uses int for size */
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
if (src_size <= 8)
return -EBADMSG;
if (src_size - 8 > INT_MAX)
return -EFBIG;
size = unaligned_read_le64(src);
if (size < 0 || (unsigned) size != unaligned_read_le64(src))
return -EFBIG;
if (dst_max > 0 && (size_t) size > dst_max)
return -ENOBUFS;
out = greedy_realloc(dst, size, 1);
if (!out)
return -ENOMEM;
r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
if (r < 0 || r != size)
return -EBADMSG;
*dst_size = size;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
size_t zstd_dstream_out_size(void) {
#if HAVE_ZSTD
if (dlopen_zstd(LOG_DEBUG) < 0)
return 0;
return sym_ZSTD_DStreamOutSize();
#else
return 0;
#endif
}
static int decompress_blob_zstd(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_ZSTD
int r;
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
/* ZSTD_getFrameContentSize() returns unsigned long long, which is not necessarily size_t, so
* keep the value in that type until we've clamped it down to something that fits size_t. */
unsigned long long size = sym_ZSTD_getFrameContentSize(src, src_size);
if (size == ZSTD_CONTENTSIZE_ERROR)
return -EBADMSG;
/* ZSTD_CONTENTSIZE_UNKNOWN is returned when the frame header doesn't record the decompressed
* size, which is the case e.g. for kernel images compressed with 'zstd'. Per the zstd
* documentation this is not an error, it just means we don't know the output size upfront and
* have to grow the output buffer as we decompress. */
/* dst_max == 0 means "no limit"; fold that into a single ceiling up front so the rest of the
* function can treat the unlimited case like any other rather than repeating the sentinel. */
size_t cap = dst_max ?: SIZE_MAX;
size_t space;
if (size != ZSTD_CONTENTSIZE_UNKNOWN)
/* The MIN clamps the (possibly 64-bit) content size to the ceiling, which is <= SIZE_MAX,
* so the result always fits size_t. */
space = MAX(sym_ZSTD_DStreamOutSize(), (size_t) MIN(size, (unsigned long long) cap));
else {
/* Start from twice the compressed size, guarding the doubling against overflow, then
* clamp to the ceiling. */
uint64_t twice = src_size <= UINT64_MAX / 2 ? src_size * 2 : UINT64_MAX;
space = MAX(sym_ZSTD_DStreamOutSize(), (size_t) MIN(twice, (uint64_t) cap));
}
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
space = MALLOC_SIZEOF_SAFE(*dst);
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
if (!dctx)
return -ENOMEM;
ZSTD_inBuffer input = {
.src = src,
.size = src_size,
};
ZSTD_outBuffer output = {
.dst = *dst,
/* Never offer the decoder more room than the cap: greedy_realloc() over-allocates, so
* without this clamp a single ZSTD_decompressStream() call could write past dst_max
* before the output.pos >= cap check below fires. */
.size = MIN(space, cap),
};
for (;;) {
size_t k = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(k))
return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
/* output.size is clamped to MIN(space, cap) here and after every grow, and zstd keeps
* output.pos <= output.size, so output.pos never exceeds cap: this fires exactly when the
* caller's limit has been filled. */
if (output.pos >= cap)
break;
/* k == 0 means the current frame was fully decoded and flushed. If no input is left
* either, the whole stream is done. This must be checked before the grow branch below:
* the frame can finish exactly as the output buffer fills, and growing then would call
* the decoder again on already-exhausted input, which it'd misread as a truncated next
* frame. */
if (k == 0 && input.pos >= input.size)
break;
/* Otherwise there is still work to do: either decoded data buffered inside the decoder
* that didn't fit (k != 0, e.g. zstd's buffered-flush mode), or another concatenated
* frame to decode (k == 0 with input left). If the output buffer is full, grow it and
* call again. Guard against the doubling overflowing size_t (only reachable with an
* uncapped dst_max once the allocation has already grown past SIZE_MAX/2). */
if (output.pos == output.size) {
if (space > SIZE_MAX / 2)
return -E2BIG;
space = MIN(2 * space, cap);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
space = MALLOC_SIZEOF_SAFE(*dst);
output.dst = *dst;
output.size = MIN(space, cap);
continue;
}
/* The output buffer still has room but the decoder stopped. If input is exhausted the
* decoder wanted more (k != 0, since k == 0 + no input broke above), i.e. the stream was
* truncated mid-frame. Otherwise input remains (a concatenated frame) and we loop. */
if (input.pos >= input.size)
return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD stream ended unexpectedly, probably corrupted.");
}
*dst_size = output.pos;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_blob_gzip(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
_cleanup_(inflateEnd_wrapper) z_stream s = {};
r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -ENOMEM;
size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.next_in = (void*) src;
s.avail_in = src_size;
s.next_out = *dst;
s.avail_out = space;
for (;;) {
size_t used;
r = sym_inflate(&s, Z_NO_FLUSH);
if (r == Z_STREAM_END)
break;
if (!IN_SET(r, Z_OK, Z_BUF_ERROR))
return -EBADMSG;
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
break;
if (dst_max > 0 && space == dst_max)
return -ENOBUFS;
used = space - s.avail_out;
space = MIN3(2 * space, dst_max ?: SIZE_MAX, UINT_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.avail_out = space - used;
s.next_out = *(uint8_t**)dst + used;
}
*dst_size = space - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_blob_bzip2(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_BZIP2
int r;
r = dlopen_bzip2(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
_cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
if (r != BZ_OK)
return -ENOMEM;
size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.next_in = (char*) src;
s.avail_in = src_size;
s.next_out = (char*) *dst;
s.avail_out = space;
for (;;) {
size_t used;
r = sym_BZ2_bzDecompress(&s);
if (r == BZ_STREAM_END)
break;
if (r != BZ_OK)
return -EBADMSG;
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
break;
if (dst_max > 0 && space == dst_max)
return -ENOBUFS;
used = space - s.avail_out;
space = MIN3(2 * space, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.avail_out = space - used;
s.next_out = (char*) *dst + used;
}
*dst_size = space - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
int decompress_blob_journal(
Compression compression,
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
switch (compression) {
case COMPRESSION_XZ:
return decompress_blob_xz(
src, src_size,
dst, dst_size, dst_max);
case COMPRESSION_LZ4:
return decompress_blob_lz4(
src, src_size,
dst, dst_size, dst_max);
case COMPRESSION_ZSTD:
return decompress_blob_zstd(
src, src_size,
dst, dst_size, dst_max);
default:
return -EPROTONOSUPPORT;
}
}
int decompress_blob(
Compression compression,
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
switch (compression) {
case COMPRESSION_GZIP:
return decompress_blob_gzip(
src, src_size,
dst, dst_size, dst_max);
case COMPRESSION_BZIP2:
return decompress_blob_bzip2(
src, src_size,
dst, dst_size, dst_max);
default:
return decompress_blob_journal(
compression,
src, src_size,
dst, dst_size, dst_max);
}
}
int decompress_zlib_raw(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_size,
int wbits) {
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
if (dst_size > UINT_MAX)
return -EFBIG;
_cleanup_(inflateEnd_wrapper) z_stream s = {
.next_in = (void*) src,
.avail_in = src_size,
.next_out = dst,
.avail_out = dst_size,
};
r = sym_inflateInit2_(&s, /* windowBits= */ wbits, ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -EIO;
r = sym_inflate(&s, Z_FINISH);
size_t produced = (uint8_t*) s.next_out - (uint8_t*) dst;
if (r != Z_STREAM_END || produced != dst_size)
return -EBADMSG;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_startswith_xz(
const void *src,
uint64_t src_size,
void **buffer,
const void *prefix,
size_t prefix_len,
uint8_t extra) {
/* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
* follow the prefix */
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_XZ
int r;
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
if (ret != LZMA_OK)
return -EBADMSG;
if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.next_in = src;
s.avail_in = src_size;
s.next_out = *buffer;
s.avail_out = allocated;
for (;;) {
ret = sym_lzma_code(&s, LZMA_FINISH);
if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
return -EBADMSG;
if (allocated - s.avail_out >= prefix_len + 1)
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
if (ret == LZMA_STREAM_END)
return 0;
s.avail_out += allocated;
if (!(greedy_realloc(buffer, allocated * 2, 1)))
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.next_out = *(uint8_t**)buffer + allocated - s.avail_out;
}
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_startswith_lz4(
const void *src,
uint64_t src_size,
void **buffer,
const void *prefix,
size_t prefix_len,
uint8_t extra) {
/* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
* follow the prefix */
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_LZ4
size_t allocated;
int r;
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
if (src_size <= 8)
return -EBADMSG;
if (src_size - 8 > INT_MAX)
return -EFBIG;
if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
r = sym_LZ4_decompress_safe_partial(
(char*)src + 8,
*buffer,
src_size - 8,
prefix_len + 1,
allocated);
/* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where just a part of the buffer is
* decompressed. But if we get a smaller amount of bytes than requested, we don't know whether there
* isn't enough data to fill the requested size or whether we just got a partial answer.
*/
if (r < 0 || (size_t) r < prefix_len + 1) {
size_t size;
if (sym_LZ4_versionNumber() >= 10803)
/* We trust that the newer lz4 decompresses the number of bytes we
* requested if available in the compressed string. */
return 0;
if (r > 0)
/* Compare what we have first, in case of mismatch we can
* shortcut the full comparison. */
if (memcmp(*buffer, prefix, r) != 0)
return 0;
/* Before version 1.8.3, lz4 always tries to decode full a "sequence",
* so in pathological cases might need to decompress the full field. */
r = decompress_blob_lz4(src, src_size, buffer, &size, 0);
if (r < 0)
return r;
if (size < prefix_len + 1)
return 0;
}
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_startswith_zstd(
const void *src,
uint64_t src_size,
void **buffer,
const void *prefix,
size_t prefix_len,
uint8_t extra) {
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_ZSTD
int r;
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
/* ZSTD_getFrameContentSize() returns unsigned long long, which is not necessarily size_t. */
unsigned long long size = sym_ZSTD_getFrameContentSize(src, src_size);
if (size == ZSTD_CONTENTSIZE_ERROR)
return -EBADMSG;
/* ZSTD_CONTENTSIZE_UNKNOWN means the frame doesn't record the decompressed size, so we can't
* shortcut here. The single decompression below only needs prefix_len + 1 bytes anyway. */
if (size != ZSTD_CONTENTSIZE_UNKNOWN && size < prefix_len + 1)
return 0; /* Decompressed text too short to match the prefix and extra */
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
if (!dctx)
return -ENOMEM;
if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
return -ENOMEM;
ZSTD_inBuffer input = {
.src = src,
.size = src_size,
};
ZSTD_outBuffer output = {
.dst = *buffer,
.size = MALLOC_SIZEOF_SAFE(*buffer),
};
size_t k;
k = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(k))
return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
if (output.pos < prefix_len + 1) {
/* The output buffer is at least prefix_len + 1 bytes, so if it isn't full the decoder
* stopped for another reason: either the frame ended (k == 0), in which case the stream
* is simply too short to match the prefix, or it's still expecting input (k != 0), which
* means the stream was truncated mid-frame. */
if (k != 0)
return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD stream ended unexpectedly, probably corrupted.");
return 0; /* Decompressed text too short to match the prefix and extra */
}
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_startswith_gzip(
const void *src,
uint64_t src_size,
void **buffer,
const void *prefix,
size_t prefix_len,
uint8_t extra) {
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
_cleanup_(inflateEnd_wrapper) z_stream s = {};
r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -EBADMSG;
if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.next_in = (void*) src;
s.avail_in = src_size;
s.next_out = *buffer;
s.avail_out = MIN(allocated, (size_t) UINT_MAX);
for (;;) {
r = sym_inflate(&s, Z_FINISH);
if (!IN_SET(r, Z_OK, Z_STREAM_END, Z_BUF_ERROR))
return -EBADMSG;
if (allocated - s.avail_out >= prefix_len + 1)
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
if (r == Z_STREAM_END)
return 0;
size_t used = allocated - s.avail_out;
if (!(greedy_realloc(buffer, allocated * 2, 1)))
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
s.next_out = *(uint8_t**)buffer + used;
}
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_startswith_bzip2(
const void *src,
uint64_t src_size,
void **buffer,
const void *prefix,
size_t prefix_len,
uint8_t extra) {
assert(src);
assert(src_size > 0);
assert(buffer);
assert(prefix);
#if HAVE_BZIP2
int r;
r = dlopen_bzip2(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
_cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
if (r != BZ_OK)
return -EBADMSG;
if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
return -ENOMEM;
size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.next_in = (char*) src;
s.avail_in = src_size;
s.next_out = *buffer;
s.avail_out = MIN(allocated, (size_t) UINT_MAX);
for (;;) {
r = sym_BZ2_bzDecompress(&s);
if (!IN_SET(r, BZ_OK, BZ_STREAM_END))
return -EBADMSG;
if (allocated - s.avail_out >= prefix_len + 1)
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
if (r == BZ_STREAM_END)
return 0;
size_t used = allocated - s.avail_out;
if (!(greedy_realloc(buffer, allocated * 2, 1)))
return -ENOMEM;
allocated = MALLOC_SIZEOF_SAFE(*buffer);
s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
s.next_out = (char*) *buffer + used;
}
#else
return -EPROTONOSUPPORT;
#endif
}
int decompress_startswith_journal(
Compression compression,
const void *src, uint64_t src_size,
void **buffer,
const void *prefix, size_t prefix_len,
uint8_t extra) {
switch (compression) {
case COMPRESSION_XZ:
return decompress_startswith_xz(src, src_size, buffer, prefix, prefix_len, extra);
case COMPRESSION_LZ4:
return decompress_startswith_lz4(src, src_size, buffer, prefix, prefix_len, extra);
case COMPRESSION_ZSTD:
return decompress_startswith_zstd(src, src_size, buffer, prefix, prefix_len, extra);
default:
return -EOPNOTSUPP;
}
}
int decompress_startswith(
Compression compression,
const void *src, uint64_t src_size,
void **buffer,
const void *prefix, size_t prefix_len,
uint8_t extra) {
switch (compression) {
case COMPRESSION_GZIP:
return decompress_startswith_gzip(src, src_size, buffer, prefix, prefix_len, extra);
case COMPRESSION_BZIP2:
return decompress_startswith_bzip2(src, src_size, buffer, prefix, prefix_len, extra);
default:
return decompress_startswith_journal(compression, src, src_size, buffer, prefix, prefix_len, extra);
}
}
int compress_stream(
Compression type,
int fdf, int fdt,
uint64_t max_bytes,
uint64_t *ret_uncompressed_size) {
_cleanup_(compressor_freep) Compressor *c = NULL;
_cleanup_free_ void *buf = NULL;
_cleanup_free_ uint8_t *input = NULL;
size_t buf_size = 0, buf_alloc = 0;
uint64_t total_in = 0, total_out = 0;
int r;
assert(fdf >= 0);
assert(fdt >= 0);
r = compressor_new(&c, type);
if (r < 0)
return r;
input = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
if (!input)
return -ENOMEM;
for (;;) {
size_t m = COMPRESS_PIPE_BUFFER_SIZE;
ssize_t n;
if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
m = (size_t) max_bytes;
n = read(fdf, input, m);
if (n < 0)
return -errno;
if (n == 0) {
r = compressor_finish(c, &buf, &buf_size, &buf_alloc);
if (r < 0)
return r;
if (buf_size > 0) {
r = loop_write(fdt, buf, buf_size);
if (r < 0)
return r;
total_out += buf_size;
}
break;
}
total_in += n;
if (max_bytes != UINT64_MAX) {
assert(max_bytes >= (uint64_t) n);
max_bytes -= n;
}
r = compressor_start(c, input, n, &buf, &buf_size, &buf_alloc);
if (r < 0)
return r;
if (buf_size > 0) {
r = loop_write(fdt, buf, buf_size);
if (r < 0)
return r;
total_out += buf_size;
}
}
if (ret_uncompressed_size)
*ret_uncompressed_size = total_in;
if (total_in == 0)
log_debug("%s compression finished (no input data)", compression_to_string(type));
else
log_debug("%s compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
compression_to_string(type), total_in, total_out, (double) total_out / total_in * 100);
return 0;
}
/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
* regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
static int should_sparse(int fd) {
struct stat st;
assert(fd >= 0);
if (fstat(fd, &st) < 0)
return -errno;
int flags = fcntl(fd, F_GETFL);
if (flags < 0)
return -errno;
return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
}
/* After sparse decompression, set the file size to the current position to account for
* trailing holes that sparse_write() created via lseek but never extended the file size for. */
static int finalize_sparse(int fd) {
off_t pos;
assert(fd >= 0);
pos = lseek(fd, 0, SEEK_CUR);
if (pos < 0)
return -errno;
if (ftruncate(fd, pos) < 0)
return -errno;
return 0;
}
/* Common helper for decompress_stream_*() wrappers */
struct decompress_stream_userdata {
int fd;
uint64_t max_bytes;
uint64_t total_out;
bool sparse;
};
static int decompress_stream_write_callback(const void *data, size_t size, void *userdata) {
struct decompress_stream_userdata *u = ASSERT_PTR(userdata);
if (u->max_bytes != UINT64_MAX) {
if (u->max_bytes < size)
return -EFBIG;
u->max_bytes -= size;
}
u->total_out += size;
if (u->sparse) {
/* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
* This is fine here since sparse mode is only used on regular files, where short
* writes and EINTR are not expected in practice. */
ssize_t k = sparse_write(u->fd, data, size, 64);
if (k < 0)
return (int) k;
return 0;
}
return loop_write(u->fd, data, size);
}
static int decompressor_new(Decompressor **ret, Compression type) {
#if HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(ret);
_cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
if (!c)
return -ENOMEM;
c->type = _COMPRESSION_INVALID;
switch (type) {
case COMPRESSION_NONE:
break;
#if HAVE_XZ
case COMPRESSION_XZ:
if (sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED) != LZMA_OK)
return -EIO;
break;
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
if (sym_LZ4F_isError(rc))
return -ENOMEM;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD:
c->d_zstd = sym_ZSTD_createDCtx();
if (!c->d_zstd)
return -ENOMEM;
break;
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip));
if (r != Z_OK)
return -EIO;
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0);
if (r != BZ_OK)
return -EIO;
break;
#endif
default:
assert_not_reached();
}
c->type = type;
c->encoding = false;
*ret = TAKE_PTR(c);
return 0;
}
int decompress_stream(
Compression type,
int fdf, int fdt,
uint64_t max_bytes) {
_cleanup_(compressor_freep) Decompressor *c = NULL;
_cleanup_free_ uint8_t *buf = NULL;
uint64_t total_in = 0;
int r;
assert(fdf >= 0);
assert(fdt >= 0);
if (type == COMPRESSION_NONE)
return -EOPNOTSUPP;
r = dlopen_compress(type, LOG_DEBUG);
if (r < 0)
return r;
r = decompressor_new(&c, type);
if (r < 0)
return r;
struct decompress_stream_userdata userdata = {
.fd = fdt,
.max_bytes = max_bytes,
.sparse = should_sparse(fdt) > 0,
};
buf = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
if (!buf)
return -ENOMEM;
for (;;) {
ssize_t n;
n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE);
if (n < 0)
return -errno;
if (n == 0)
break;
total_in += n;
r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata);
if (r < 0)
return r;
}
if (total_in == 0)
return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "%s decompression failed: no data read",
compression_to_string(type));
if (userdata.sparse) {
r = finalize_sparse(fdt);
if (r < 0)
return r;
}
log_debug("%s decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
compression_to_string(type), total_in, userdata.total_out,
(double) userdata.total_out / total_in * 100);
return 0;
}
int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
Compression c = compression_from_filename(filename);
if (c == COMPRESSION_NONE)
return -EPROTONOSUPPORT;
return decompress_stream(c, fdf, fdt, max_bytes);
}
/* Push-based streaming compression/decompression context API */
Compressor* compressor_free(Compressor *c) {
if (!c)
return NULL;
switch (c->type) {
#if HAVE_XZ
case COMPRESSION_XZ:
sym_lzma_end(&c->xz);
break;
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4:
if (c->encoding) {
sym_LZ4F_freeCompressionContext(c->c_lz4);
c->c_lz4 = NULL;
c->lz4_header = mfree(c->lz4_header);
} else {
sym_LZ4F_freeDecompressionContext(c->d_lz4);
c->d_lz4 = NULL;
}
break;
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD:
if (c->encoding) {
sym_ZSTD_freeCCtx(c->c_zstd);
c->c_zstd = NULL;
} else {
sym_ZSTD_freeDCtx(c->d_zstd);
c->d_zstd = NULL;
}
break;
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
if (c->encoding)
sym_deflateEnd(&c->gzip);
else
sym_inflateEnd(&c->gzip);
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
if (c->encoding)
sym_BZ2_bzCompressEnd(&c->bzip2);
else
sym_BZ2_bzDecompressEnd(&c->bzip2);
break;
#endif
default:
break;
}
return mfree(c);
}
Compression compressor_type(const Compressor *c) {
return c ? c->type : _COMPRESSION_INVALID;
}
int decompressor_detect(Decompressor **decompressor, const void *data, size_t size) {
int r;
assert(decompressor);
if (*decompressor)
return 1;
if (size < COMPRESSION_MAGIC_BYTES_MAX)
return 0;
assert(data);
Compression type = compression_detect_from_magic(data);
if (type == _COMPRESSION_INVALID)
type = COMPRESSION_NONE;
log_debug("Detected compression type: %s", compression_to_string(type));
r = dlopen_compress(type, LOG_DEBUG);
if (r < 0)
return r;
r = decompressor_new(decompressor, type);
if (r < 0)
return r;
return 1;
}
int decompressor_force_off(Decompressor **decompressor) {
assert(decompressor);
*decompressor = compressor_free(*decompressor);
return decompressor_new(decompressor, COMPRESSION_NONE);
}
int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata) {
int r;
assert(c);
assert(callback);
if (c->encoding)
return -EINVAL;
if (size == 0)
return 1;
assert(data);
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
_cleanup_free_ uint8_t *buffer = NULL;
if (c->type != COMPRESSION_NONE) {
buffer = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
if (!buffer)
return -ENOMEM;
}
#endif
switch (c->type) {
case COMPRESSION_NONE:
r = callback(data, size, userdata);
if (r < 0)
return r;
break;
#if HAVE_XZ
case COMPRESSION_XZ:
c->xz.next_in = data;
c->xz.avail_in = size;
while (c->xz.avail_in > 0) {
c->xz.next_out = buffer;
c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN);
if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
return -EBADMSG;
if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata);
if (r < 0)
return r;
}
}
break;
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
const uint8_t *src = data;
size_t src_remaining = size;
while (src_remaining > 0) {
size_t produced = COMPRESS_PIPE_BUFFER_SIZE;
size_t consumed = src_remaining;
size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL);
if (sym_LZ4F_isError(rc))
return -EBADMSG;
if (consumed == 0 && produced == 0)
break; /* No progress possible with current input */
src += consumed;
src_remaining -= consumed;
if (produced > 0) {
r = callback(buffer, produced, userdata);
if (r < 0)
return r;
}
}
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {
.src = (void*) data,
.size = size,
};
while (input.pos < input.size) {
ZSTD_outBuffer output = {
.dst = buffer,
.size = COMPRESS_PIPE_BUFFER_SIZE,
};
size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input);
if (sym_ZSTD_isError(res))
return -EBADMSG;
if (output.pos > 0) {
r = callback(output.dst, output.pos, userdata);
if (r < 0)
return r;
}
}
break;
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
if (size > UINT_MAX)
return -EFBIG;
c->gzip.next_in = (void*) data;
c->gzip.avail_in = size;
while (c->gzip.avail_in > 0) {
c->gzip.next_out = buffer;
c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
int zr = sym_inflate(&c->gzip, Z_NO_FLUSH);
if (!IN_SET(zr, Z_OK, Z_STREAM_END))
return -EBADMSG;
if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata);
if (r < 0)
return r;
}
if (zr == Z_STREAM_END)
break;
}
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
if (size > UINT_MAX)
return -EFBIG;
c->bzip2.next_in = (char*) data;
c->bzip2.avail_in = size;
while (c->bzip2.avail_in > 0) {
c->bzip2.next_out = (char*) buffer;
c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
int bzr = sym_BZ2_bzDecompress(&c->bzip2);
if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END))
return -EBADMSG;
if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata);
if (r < 0)
return r;
}
if (bzr == BZ_STREAM_END)
break;
}
break;
#endif
default:
assert_not_reached();
}
return 1;
}
int compressor_new(Compressor **ret, Compression type) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(ret);
_cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1);
if (!c)
return -ENOMEM;
c->type = _COMPRESSION_INVALID;
/* Set encoding early so that compressor_freep calls the correct cleanup (compression vs
* decompression) if any operation in the switch fails after setting c->type. This is safe
* because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */
c->encoding = true;
switch (type) {
#if HAVE_XZ
case COMPRESSION_XZ: {
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (xzr != LZMA_OK)
return -EIO;
c->type = COMPRESSION_XZ;
break;
}
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION);
if (sym_LZ4F_isError(rc))
return -ENOMEM;
c->type = COMPRESSION_LZ4;
/* Generate the frame header and stash it for the first compressor_start call */
size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences);
c->lz4_header = malloc(header_bound);
if (!c->lz4_header)
return -ENOMEM;
c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences);
if (sym_LZ4F_isError(c->lz4_header_size))
return -EINVAL;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD:
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
c->c_zstd = sym_ZSTD_createCCtx();
if (!c->c_zstd)
return -ENOMEM;
c->type = COMPRESSION_ZSTD;
size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
if (sym_ZSTD_isError(z))
return -EIO;
z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1);
if (sym_ZSTD_isError(z))
log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
break;
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
r = sym_deflateInit2_(&c->gzip,
Z_DEFAULT_COMPRESSION,
/* method= */ Z_DEFLATED,
/* windowBits= */ ZLIB_WBITS_GZIP,
/* memLevel= */ 8,
/* strategy= */ Z_DEFAULT_STRATEGY,
ZLIB_VERSION, (int) sizeof(c->gzip));
if (r != Z_OK)
return -EIO;
c->type = COMPRESSION_GZIP;
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
r = dlopen_bzip2(LOG_DEBUG);
if (r < 0)
return r;
r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0);
if (r != BZ_OK)
return -EIO;
c->type = COMPRESSION_BZIP2;
break;
#endif
case COMPRESSION_NONE:
c->type = COMPRESSION_NONE;
break;
default:
return -EOPNOTSUPP;
}
*ret = TAKE_PTR(c);
return 0;
}
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) {
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE);
if (*buffer_allocated >= need)
return 0;
if (!greedy_realloc(buffer, need, 1))
return -ENOMEM;
*buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer);
return 1;
}
#endif
int compressor_start(
Compressor *c,
const void *data,
size_t size,
void **buffer,
size_t *buffer_size,
size_t *buffer_allocated) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(c);
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
if (!c->encoding)
return -EINVAL;
if (size == 0)
return 0;
assert(data);
*buffer_size = 0;
switch (c->type) {
#if HAVE_XZ
case COMPRESSION_XZ:
c->xz.next_in = data;
c->xz.avail_in = size;
while (c->xz.avail_in > 0) {
lzma_ret lzr;
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
c->xz.avail_out = *buffer_allocated - *buffer_size;
lzr = sym_lzma_code(&c->xz, LZMA_RUN);
if (lzr != LZMA_OK)
return -EIO;
*buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
}
break;
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
/* Prepend any stashed frame header from compressor_new */
if (c->lz4_header_size > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size);
if (r < 0)
return r;
memcpy(*buffer, c->lz4_header, c->lz4_header_size);
*buffer_size = c->lz4_header_size;
c->lz4_header = mfree(c->lz4_header);
c->lz4_header_size = 0;
}
size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences);
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound);
if (r < 0)
return r;
size_t n = sym_LZ4F_compressUpdate(c->c_lz4,
(uint8_t*) *buffer + *buffer_size,
*buffer_allocated - *buffer_size,
data, size, NULL);
if (sym_LZ4F_isError(n))
return -EIO;
*buffer_size += n;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {
.src = data,
.size = size,
};
while (input.pos < input.size) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
ZSTD_outBuffer output = {
.dst = ((uint8_t *) *buffer + *buffer_size),
.size = *buffer_allocated - *buffer_size,
};
size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
if (sym_ZSTD_isError(res))
return -EIO;
*buffer_size += output.pos;
}
break;
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
if (size > UINT_MAX)
return -EFBIG;
c->gzip.next_in = (void*) data;
c->gzip.avail_in = size;
while (c->gzip.avail_in > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
c->gzip.avail_out = avail;
r = sym_deflate(&c->gzip, Z_NO_FLUSH);
if (r != Z_OK)
return -EIO;
*buffer_size += avail - c->gzip.avail_out;
}
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
if (size > UINT_MAX)
return -EFBIG;
c->bzip2.next_in = (void*) data;
c->bzip2.avail_in = size;
while (c->bzip2.avail_in > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
c->bzip2.avail_out = avail;
r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN);
if (r != BZ_RUN_OK)
return -EIO;
*buffer_size += avail - c->bzip2.avail_out;
}
break;
#endif
case COMPRESSION_NONE:
if (*buffer_allocated < size) {
void *p;
p = realloc(*buffer, size);
if (!p)
return -ENOMEM;
*buffer = p;
*buffer_allocated = size;
}
memcpy(*buffer, data, size);
*buffer_size = size;
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(c);
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
if (!c->encoding)
return -EINVAL;
*buffer_size = 0;
switch (c->type) {
#if HAVE_XZ
case COMPRESSION_XZ: {
lzma_ret lzr;
c->xz.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
c->xz.avail_out = *buffer_allocated - *buffer_size;
lzr = sym_lzma_code(&c->xz, LZMA_FINISH);
if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
return -EIO;
*buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
} while (lzr != LZMA_STREAM_END);
break;
}
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences);
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound);
if (r < 0)
return r;
size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL);
if (sym_LZ4F_isError(n))
return -EIO;
*buffer_size = n;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {};
size_t res;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
ZSTD_outBuffer output = {
.dst = ((uint8_t *) *buffer + *buffer_size),
.size = *buffer_allocated - *buffer_size,
};
res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
if (sym_ZSTD_isError(res))
return -EIO;
*buffer_size += output.pos;
} while (res != 0);
break;
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
c->gzip.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
c->gzip.avail_out = avail;
r = sym_deflate(&c->gzip, Z_FINISH);
if (!IN_SET(r, Z_OK, Z_STREAM_END))
return -EIO;
*buffer_size += avail - c->gzip.avail_out;
} while (r != Z_STREAM_END);
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
c->bzip2.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
c->bzip2.avail_out = avail;
r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH);
if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
return -EIO;
*buffer_size += avail - c->bzip2.avail_out;
} while (r != BZ_STREAM_END);
break;
#endif
case COMPRESSION_NONE:
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink