tdf#137308 - Add zstd build support and decompression classes

Integrate build system support for the Zstandard (zstd) library and
implement the corresponding C++ decompression stream classes, enabling
future use of zstd-compressed streams in ZIP packages.

Build System (Part 2a):
- Add `--with-system-zstd=[yes|no|auto]` configure option.
- Implement system library checks (`zstd.h`, `libzstd`) in configure.ac.
- Define rules (`external/zstd/*`) for downloading and building the
  internal zstd static library (decompression components only).
- Set up necessary CFLAGS/LIBS via configure.ac and config_host.mk.in.
- Configure linking via RepositoryExternal.mk (`gb_LinkTarget__use_zstd`).
- Update auxiliary files (download.lst, distro-configs, etc.).

C++ Implementation (Part 2b):
- Implement `InflateZstd` (in `include/package`) and `InflaterBytesZstd`
  (in `package/inc`) inheriting from `Inflater`/`InflaterBytes`.
- Use the zstd streaming API (`ZSTD_decompressStream`) following the
  existing single-call pattern established by the zlib implementation.
- Manage context (`ZSTD_DCtx`) and input buffer (`ZSTD_inBuffer`) state.
- Handle errors, stream completion (`ret == 0`), and input truncation.
- Add new classes to `package/Library_package2.mk` build.

This completes the infrastructure for zstd decompression support.
The next step (Part 2c) involves adding logic to select the
appropriate Inflater based on the ZIP entry's compression method ID.

Change-Id: Ia673f3f19b6a751ba5225394fcf8c3f145c4ae63
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/184243
Tested-by: Jenkins
Reviewed-by: Caolán McNamara <caolan.mcnamara@collabora.com>
This commit is contained in:
Akshay Kumar Dubey 2025-04-16 02:05:16 +05:30 committed by Caolán McNamara
parent 55169df2f6
commit b410749d30
21 changed files with 440 additions and 0 deletions

View File

@ -240,6 +240,7 @@ $(WORKDIR)/download: $(BUILDDIR)/config_$(gb_Side).mk $(SRCDIR)/download.lst $(S
$(call fetch_Optional,WPS,WPS_TARBALL) \
$(call fetch_Optional,XSLTML,XSLTML_TARBALL) \
$(call fetch_Optional,ZLIB,ZLIB_TARBALL) \
$(call fetch_Optional,ZSTD,ZSTD_TARBALL) \
$(call fetch_Optional,ZMF,ZMF_TARBALL) \
,$(call fetch_Download_item,https://dev-www.libreoffice.org/src,$(item)))
$(foreach item, \

View File

@ -409,6 +409,23 @@ endef
endif # SYSTEM_ZLIB
ifneq ($(SYSTEM_ZSTD),)
define gb_LinkTarget__use_zstd
$(call gb_LinkTarget_add_libs,$(1),-lzstd)
endef
gb_ExternalProject__use_zstd :=
else
define gb_LinkTarget__use_zstd
$(call gb_LinkTarget_set_include,$(1),$(ZSTD_CFLAGS) $$(INCLUDE))
$(call gb_LinkTarget_use_static_libraries,$(1),zstd)
endef
define gb_ExternalProject__use_zstd
$(call gb_ExternalProject_use_static_libraries,$(1),zstd)
endef
endif
ifneq ($(SYSTEM_LIBJPEG),)

View File

@ -53,6 +53,7 @@ curl --no-progress-meter -S \
-C - -O https://dev-www.libreoffice.org/src/$ZMF_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$PIXMAN_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$ZLIB_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$ZSTD_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$MDDS_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$OPENSSL_TARBALL \
-C - -O https://dev-www.libreoffice.org/src/$LANGTAGREG_TARBALL \

View File

@ -736,6 +736,7 @@ SYSTEM_WPG=@SYSTEM_WPG@
SYSTEM_WPS=@SYSTEM_WPS@
SYSTEM_XMLSEC=@SYSTEM_XMLSEC@
SYSTEM_ZLIB=@SYSTEM_ZLIB@
SYSTEM_ZSTD=@SYSTEM_ZSTD@
SYSTEM_ZMF=@SYSTEM_ZMF@
export SYSTEMD_ESCAPE=@SYSTEMD_ESCAPE@
export SYSTEMD_RUN=@SYSTEMD_RUN@
@ -803,6 +804,8 @@ export XSLTPROC=@XSLTPROC@
export XVFB_RUN=@XVFB_RUN@
export ZLIB_CFLAGS=$(gb_SPACE)@ZLIB_CFLAGS@
export ZLIB_LIBS=$(gb_SPACE)@ZLIB_LIBS@
ZSTD_CFLAGS=$(gb_SPACE)@ZSTD_CFLAGS@
ZSTD_LIBS=$(gb_SPACE)@ZSTD_LIBS@
export ZMF_CFLAGS=$(gb_SPACE)@ZMF_CFLAGS@
export ZMF_LIBS=$(gb_SPACE)@ZMF_LIBS@
export GET_TASK_ALLOW_ENTITLEMENT=@GET_TASK_ALLOW_ENTITLEMENT@

View File

@ -2439,6 +2439,11 @@ AC_ARG_WITH(system-zlib,
[Use zlib already on system.]),,
[with_system_zlib=auto])
AC_ARG_WITH(system-zstd,
AS_HELP_STRING([--with-system-zstd=@<:@yes|no|auto@:>@],
[Use zstd already on system [default=auto].]),,
[with_system_zstd=auto])
AC_ARG_WITH(system-jpeg,
AS_HELP_STRING([--with-system-jpeg],
[Use jpeg already on system.]),,
@ -9857,6 +9862,52 @@ AC_SUBST(ZLIB_CFLAGS)
AC_SUBST(ZLIB_LIBS)
AC_SUBST(SYSTEM_ZLIB)
dnl ===================================================================
dnl Check for system zstd
dnl ===================================================================
if test "x$with_system_zstd" = "xauto"; then
case "$_os" in
WINNT) with_system_zstd="$with_system_libs" ;; # follow system-libs on Windows
*) with_system_zstd="$with_system_libs" ;; # follow system-libs on Unix
esac
fi
AC_MSG_CHECKING([which zstd to use])
case "$with_system_zstd" in
yes)
AC_MSG_RESULT([system])
SYSTEM_ZSTD=TRUE
AC_CHECK_HEADER([zstd.h], [],
[AC_MSG_ERROR([zstd.h not found. Install system zstd.])])
AC_CHECK_LIB([zstd], [ZSTD_createDStream],
[ ZSTD_LIBS="-lzstd"
ZSTD_CFLAGS="" ],
[ AC_MSG_ERROR([system zstd library not found or broken.]) ])
;;
""|no)
AC_MSG_RESULT([bundled])
SYSTEM_ZSTD=
ZSTD_LIBS=""
ZSTD_CFLAGS="-I\${WORKDIR}/UnpackedTarball/zstd/lib \
-I\${WORKDIR}/UnpackedTarball/zstd/lib/common \
-I\${WORKDIR}/UnpackedTarball/zstd/lib/decompress"
BUILD_TYPE="$BUILD_TYPE ZSTD"
;;
*)
AC_MSG_ERROR([Invalid value for --with-system-zstd: $with_system_zstd])
;;
esac
AC_SUBST(SYSTEM_ZSTD)
AC_SUBST(ZSTD_LIBS)
AC_SUBST(ZSTD_CFLAGS)
AC_SUBST(BUILD_TYPE)
dnl ===================================================================
dnl Check for system jpeg
dnl ===================================================================

View File

@ -6,6 +6,7 @@
--with-system-dicts
--with-myspell-dicts
--with-system-zlib
--with-system-zstd
--disable-poppler
--enable-cairo-rgba
--without-system-cairo

View File

@ -3,6 +3,7 @@
--with-system-dicts
--with-myspell-dicts
--with-system-zlib
--with-system-zstd
--without-system-poppler
--without-system-openssl
--without-system-libpng

View File

@ -1,6 +1,7 @@
--with-system-dicts
--with-myspell-dicts
--with-system-zlib
--with-system-zstd
--without-system-poppler
--without-system-openssl
--without-system-libpng

View File

@ -37,6 +37,7 @@
--with-linker-hash-style=both
--with-system-dicts
--with-system-zlib
--with-system-zstd
--with-theme=colibre
--without-branding
--without-help

View File

@ -26,6 +26,7 @@
--with-system-poppler
--with-system-redland
--with-system-zlib
--with-system-zstd
--with-vendor=The OpenBSD project
--without-junit
--without-system-libwpd

View File

@ -680,6 +680,11 @@ ZLIB_TARBALL := zlib-1.3.1.tar.xz
# three static lines
# so that git cherry-pick
# will not run into conflicts
ZSTD_SHA256SUM := eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
ZSTD_TARBALL := zstd-1.5.7.tar.gz
# three static lines
# so that git cherry-pick
# will not run into conflicts
ZMF_SHA256SUM := 27051a30cb057fdb5d5de65a1f165c7153dc76e27fe62251cbb86639eb2caf22
ZMF_TARBALL := libzmf-0.0.2.tar.xz
# three static lines

View File

@ -101,6 +101,7 @@ $(eval $(call gb_Module_add_moduledirs,external,\
$(call gb_Helper_optional,WPS,libwps) \
$(call gb_Helper_optional,XSLTML,xsltml) \
$(call gb_Helper_optional,ZLIB,zlib) \
$(call gb_Helper_optional,ZSTD,zstd) \
$(call gb_Helper_optional,ZMF,libzmf) \
))

14
external/zstd/Makefile vendored Normal file
View File

@ -0,0 +1,14 @@
# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
module_directory:=$(dir $(realpath $(firstword $(MAKEFILE_LIST))))
include $(module_directory)/../../solenv/gbuild/partial_build.mk
# vim: set noet sw=4 ts=4:

17
external/zstd/Module_zstd.mk vendored Normal file
View File

@ -0,0 +1,17 @@
# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
$(eval $(call gb_Module_Module,zstd))
$(eval $(call gb_Module_add_targets,zstd,\
StaticLibrary_zstd \
UnpackedTarball_zstd \
))
# vim: set noet sw=4 ts=4:

41
external/zstd/StaticLibrary_zstd.mk vendored Normal file
View File

@ -0,0 +1,41 @@
# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
$(eval $(call gb_StaticLibrary_StaticLibrary,zstd))
$(eval $(call gb_StaticLibrary_use_unpacked,zstd,zstd))
$(eval $(call gb_StaticLibrary_set_warnings_disabled,zstd))
$(eval $(call gb_StaticLibrary_set_include,zstd,\
-I$(gb_UnpackedTarball_workdir)/zstd/lib \
-I$(gb_UnpackedTarball_workdir)/zstd/lib/common \
-I$(gb_UnpackedTarball_workdir)/zstd/lib/decompress \
$$(INCLUDE) \
))
$(eval $(call gb_StaticLibrary_add_generated_cobjects,zstd,\
UnpackedTarball/zstd/lib/common/entropy_common \
UnpackedTarball/zstd/lib/common/error_private \
UnpackedTarball/zstd/lib/common/fse_decompress \
UnpackedTarball/zstd/lib/common/xxhash \
UnpackedTarball/zstd/lib/common/zstd_common \
UnpackedTarball/zstd/lib/decompress/huf_decompress \
UnpackedTarball/zstd/lib/decompress/zstd_ddict \
UnpackedTarball/zstd/lib/decompress/zstd_decompress \
UnpackedTarball/zstd/lib/decompress/zstd_decompress_block \
))
$(eval $(call gb_StaticLibrary_add_cflags,zstd,-DZSTD_DISABLE_ASM))
ifeq ($(ENABLE_DEBUG),TRUE)
$(eval $(call gb_StaticLibrary_add_cflags,zstd,-DZSTD_DEBUG=1))
endif
# vim: set noet sw=4 ts=4:

14
external/zstd/UnpackedTarball_zstd.mk vendored Normal file
View File

@ -0,0 +1,14 @@
# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
#
# This file is part of the LibreOffice project.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
$(eval $(call gb_UnpackedTarball_UnpackedTarball,zstd))
$(eval $(call gb_UnpackedTarball_set_tarball,zstd,$(ZSTD_TARBALL)))
# vim: set noet sw=4 ts=4:

View File

@ -0,0 +1,48 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <memory>
#include <package/Inflater.hxx>
#include <zstd.h>
#include <zstd_errors.h>
namespace ZipUtils
{
class DLLPUBLIC_PACKAGE InflateZstd : public Inflater
{
private:
bool bFinished;
bool bNeedDict;
sal_Int32 nLastInflateError;
css::uno::Sequence<sal_Int8> sInBuffer;
ZSTD_DCtx* pDCtx;
ZSTD_inBuffer inBuffer;
bool bStreamInitialized;
sal_Int32 doInflateBytes(css::uno::Sequence<sal_Int8>& rBuffer, sal_Int32 nNewOffset,
sal_Int32 nNewLength);
public:
explicit InflateZstd(bool bNoWrap = false);
virtual ~InflateZstd() override;
virtual void setInput(const css::uno::Sequence<sal_Int8>& rBuffer) override;
virtual bool needsDictionary() const override { return bNeedDict; }
virtual bool finished() const override { return bFinished; }
virtual sal_Int32 doInflateSegment(css::uno::Sequence<sal_Int8>& rBuffer, sal_Int32 nNewOffset,
sal_Int32 nNewLength) override;
virtual void end() override final;
virtual sal_Int32 getLastInflateError() const override { return nLastInflateError; }
};
} // namespace ZipUtils
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */

View File

@ -43,6 +43,7 @@ $(eval $(call gb_Library_use_libraries,package2,\
$(eval $(call gb_Library_use_externals,package2,\
argon2 \
zlib \
zstd \
))
$(eval $(call gb_Library_add_exception_objects,package2,\
@ -56,7 +57,9 @@ $(eval $(call gb_Library_add_exception_objects,package2,\
package/source/zipapi/CRC32 \
package/source/zipapi/Deflater \
package/source/zipapi/InflaterBytesZlib \
package/source/zipapi/InflaterBytesZstd \
package/source/zipapi/InflateZlib \
package/source/zipapi/InflateZstd \
package/source/zipapi/sha1context \
package/source/zipapi/ThreadedDeflater \
package/source/zipapi/XBufferedThreadedStream \

View File

@ -0,0 +1,43 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#pragma once
#include <memory>
#include <package/Inflater.hxx>
#include <zstd.h>
namespace ZipUtils
{
class DLLPUBLIC_PACKAGE InflaterBytesZstd : public InflaterBytes
{
private:
bool bFinished;
const sal_Int8* sInBuffer;
ZSTD_DCtx* pDCtx;
sal_Int32 nLastInflateError;
ZSTD_inBuffer inBuffer;
bool bStreamInitialized;
sal_Int32 doInflateBytes(sal_Int8* pOutBuffer, sal_Int32 nNewOffset, sal_Int32 nNewLength);
public:
InflaterBytesZstd();
virtual ~InflaterBytesZstd() override;
virtual void setInput(const sal_Int8* pBuffer, sal_Int32 nLen) override;
virtual bool finished() const override { return bFinished; }
virtual sal_Int32 doInflateSegment(sal_Int8* pOutBuffer, sal_Int32 nBufLen,
sal_Int32 nNewOffset, sal_Int32 nNewLength) override;
virtual void end() override final;
};
} // namespace ZipUtils
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */

View File

@ -0,0 +1,94 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#include <package/InflateZstd.hxx>
#include <string.h>
using namespace com::sun::star::uno;
using namespace ZipUtils;
InflateZstd::InflateZstd(bool)
: bFinished(false)
, bNeedDict(false)
, nLastInflateError(0)
, pDCtx(ZSTD_createDCtx())
, bStreamInitialized(false)
{
if (!pDCtx)
{
nLastInflateError = static_cast<sal_Int32>(ZSTD_error_memory_allocation);
}
inBuffer = { nullptr, 0, 0 };
}
InflateZstd::~InflateZstd() { end(); }
void InflateZstd::setInput(const Sequence<sal_Int8>& rBuffer)
{
if (!pDCtx)
{
bStreamInitialized = false;
return;
}
sInBuffer = rBuffer;
inBuffer.src = sInBuffer.getConstArray();
inBuffer.size = sInBuffer.getLength();
inBuffer.pos = 0;
bStreamInitialized = true;
}
sal_Int32 InflateZstd::doInflateSegment(Sequence<sal_Int8>& rBuffer, sal_Int32 nNewOffset,
sal_Int32 nNewLength)
{
if (nNewOffset < 0 || nNewLength < 0 || nNewOffset + nNewLength > rBuffer.getLength())
return 0;
return doInflateBytes(rBuffer, nNewOffset, nNewLength);
}
void InflateZstd::end()
{
if (pDCtx)
{
ZSTD_freeDCtx(pDCtx);
pDCtx = nullptr;
}
bStreamInitialized = false;
inBuffer = { nullptr, 0, 0 };
}
sal_Int32 InflateZstd::doInflateBytes(Sequence<sal_Int8>& rBuffer, sal_Int32 nNewOffset,
sal_Int32 nNewLength)
{
if (bFinished)
{
return 0;
}
if (!pDCtx || !bStreamInitialized)
{
nLastInflateError = 1;
return 0;
}
nLastInflateError = 0;
ZSTD_outBuffer outBuffer
= { rBuffer.getArray() + nNewOffset, static_cast<size_t>(nNewLength), 0 };
size_t ret = ZSTD_decompressStream(pDCtx, &outBuffer, &inBuffer);
if (ZSTD_isError(ret))
{
nLastInflateError = static_cast<sal_Int32>(ret);
ZSTD_DCtx_reset(pDCtx, ZSTD_reset_session_only);
return 0;
}
if (ret == 0)
bFinished = true;
return static_cast<sal_Int32>(outBuffer.pos);
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */

View File

@ -0,0 +1,82 @@
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#include <InflaterBytesZstd.hxx>
#include <string.h>
using namespace ZipUtils;
InflaterBytesZstd::InflaterBytesZstd()
: bFinished(false)
, sInBuffer(nullptr)
, pDCtx(ZSTD_createDCtx())
, nLastInflateError(0)
, bStreamInitialized(false)
{
inBuffer = { nullptr, 0, 0 };
}
InflaterBytesZstd::~InflaterBytesZstd() { end(); }
void InflaterBytesZstd::setInput(const sal_Int8* rBuffer, sal_Int32 nBufLen)
{
sInBuffer = rBuffer;
inBuffer.src = sInBuffer;
inBuffer.size = nBufLen;
inBuffer.pos = 0;
bStreamInitialized = true;
}
sal_Int32 InflaterBytesZstd::doInflateSegment(sal_Int8* pOutBuffer, sal_Int32 nBufLen,
sal_Int32 nNewOffset, sal_Int32 nNewLength)
{
if (nNewOffset < 0 || nNewLength < 0 || nNewOffset + nNewLength > nBufLen)
return 0;
return doInflateBytes(pOutBuffer, nNewOffset, nNewLength);
}
void InflaterBytesZstd::end()
{
if (pDCtx)
{
ZSTD_freeDCtx(pDCtx);
pDCtx = nullptr;
}
bStreamInitialized = false;
inBuffer = { nullptr, 0, 0 };
}
sal_Int32 InflaterBytesZstd::doInflateBytes(sal_Int8* pOutBuffer, sal_Int32 nNewOffset,
sal_Int32 nNewLength)
{
if (!pDCtx || !bStreamInitialized)
return 0;
ZSTD_outBuffer outBuffer = { pOutBuffer + nNewOffset, static_cast<size_t>(nNewLength), 0 };
size_t ret = ZSTD_decompressStream(pDCtx, &outBuffer, &inBuffer);
if (ZSTD_isError(ret))
{
nLastInflateError = static_cast<sal_Int32>(ret);
ZSTD_DCtx_reset(pDCtx, ZSTD_reset_session_only);
return 0;
}
if (ret != 0)
{
ZSTD_DCtx_reset(pDCtx, ZSTD_reset_session_only);
return static_cast<sal_Int32>(outBuffer.pos);
}
if (ret == 0)
bFinished = true;
nLastInflateError = 0;
return static_cast<sal_Int32>(outBuffer.pos);
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */