Add support for scanning Zstandard (zstd) compressed files

CMakeLists.txt

@@ -514,6 +514,8 @@ find_package(ZLIB REQUIRED)
 
 find_package(BZip2 REQUIRED)
 
+find_package(ZSTD REQUIRED)
+
 # Disable CMAKE_FIND_PACKAGE_PREFER_CONFIG, temporarily, because
 # we don't presently support the using libxml2's Config.cmake
 set(PACKAGE_PREFER_CONFIG_BAK ${CMAKE_FIND_PACKAGE_PREFER_CONFIG})

INSTALL.md

@@ -39,6 +39,7 @@ configuration options.
     - [`libcheck`](#libcheck)
     - [`bzip2`](#bzip2)
     - [`zlib`](#zlib)
+    - [`zstd`](#zstd)
     - [`libxml2`](#libxml2)
     - [`libpcre2`](#libpcre2)
     - [`openssl` (`libcrypto`, `libssl`)](#openssl-libcrypto-libssl)
@@ -114,6 +115,7 @@ libclamav requires these library dependencies:
 
 - `libbz2` / `bzip2`
 - `libz` / `zlib`
+- `libzstd` / `zstd`
 - `libxml2`
 - `libpcre2`
 - `openssl`
@@ -569,6 +571,13 @@ But if you:
   -D ZLIB_LIBRARY="_filepath of zlib library_"
 ```
 
+### `zstd`
+
+```sh
+  -D ZSTD_INCLUDE_DIR="_filepath of zstd header directory_"
+  -D ZSTD_LIBRARY="_filepath of zstd library_"
+```
+
 ### `libxml2`
 
 ```sh

cmake/FindZSTD.cmake

@@ -0,0 +1,116 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindZSTD
+-------
+
+Finds the Zstandard (zstd) library.
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module provides the following imported targets, if found:
+
+``ZSTD::zstd``
+  The Zstandard library
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``ZSTD_FOUND``
+  True if the system has the zstd library.
+``ZSTD_VERSION``
+  The version of the zstd library which was found.
+``ZSTD_INCLUDE_DIRS``
+  Include directories needed to use zstd.
+``ZSTD_LIBRARIES``
+  Libraries needed to link to zstd.
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``ZSTD_INCLUDE_DIR``
+  The directory containing ``zstd.h``.
+``ZSTD_LIBRARY``
+  The path to the zstd library.
+
+#]=======================================================================]
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_ZSTD QUIET libzstd)
+
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  PATHS ${PC_ZSTD_INCLUDE_DIRS}
+  PATH_SUFFIXES zstd include/zstd
+)
+
+if(ZSTD_LIBRARY)
+  set(ZSTD_LIBRARIES "${ZSTD_LIBRARY}")
+endif()
+
+if(NOT ZSTD_LIBRARIES)
+  find_library(ZSTD_LIBRARY_RELEASE
+    NAMES zstd NAMES_PER_DIR ${PC_ZSTD_LIBRARY_DIRS} PATH_SUFFIXES lib)
+  find_library(ZSTD_LIBRARY_DEBUG
+    NAMES zstdd NAMES_PER_DIR ${PC_ZSTD_LIBRARY_DIRS} PATH_SUFFIXES lib)
+
+  include(SelectLibraryConfigurations)
+  SELECT_LIBRARY_CONFIGURATIONS(ZSTD)
+else()
+  file(TO_CMAKE_PATH "${ZSTD_LIBRARIES}" ZSTD_LIBRARIES)
+endif()
+
+set(ZSTD_VERSION ${PC_ZSTD_VERSION})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ZSTD
+  FOUND_VAR ZSTD_FOUND
+  REQUIRED_VARS
+    ZSTD_LIBRARIES
+    ZSTD_INCLUDE_DIR
+  VERSION_VAR ZSTD_VERSION
+)
+
+if(ZSTD_FOUND)
+  set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+  set(ZSTD_DEFINITIONS ${PC_ZSTD_CFLAGS_OTHER})
+
+  if(NOT TARGET ZSTD::zstd)
+    add_library(ZSTD::zstd UNKNOWN IMPORTED)
+    set_target_properties(ZSTD::zstd PROPERTIES
+      INTERFACE_COMPILE_OPTIONS "${PC_ZSTD_CFLAGS_OTHER}"
+      INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIRS}")
+
+    if(ZSTD_LIBRARY_RELEASE)
+      set_property(TARGET ZSTD::zstd APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS RELEASE)
+      set_target_properties(ZSTD::zstd PROPERTIES
+        IMPORTED_LOCATION_RELEASE "${ZSTD_LIBRARY_RELEASE}"
+      )
+    endif()
+
+    if(ZSTD_LIBRARY_DEBUG)
+      set_property(TARGET ZSTD::zstd APPEND PROPERTY
+        IMPORTED_CONFIGURATIONS DEBUG)
+      set_target_properties(ZSTD::zstd PROPERTIES
+        IMPORTED_LOCATION_DEBUG "${ZSTD_LIBRARY_DEBUG}"
+      )
+    endif()
+
+    if(NOT ZSTD_LIBRARY_RELEASE AND NOT ZSTD_LIBRARY_DEBUG)
+      set_property(TARGET ZSTD::zstd APPEND PROPERTY
+        IMPORTED_LOCATION "${ZSTD_LIBRARY}")
+    endif()
+  endif()
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY
+)

libclamav/CMakeLists.txt

@@ -446,6 +446,7 @@ if(ENABLE_SHARED_LIB)
             OpenSSL::Crypto
             ZLIB::ZLIB
             BZip2::BZip2
+            ZSTD::zstd
             PCRE2::pcre2
             LibXml2::LibXml2
             JSONC::jsonc )
@@ -570,6 +571,7 @@ if(ENABLE_STATIC_LIB)
             OpenSSL::Crypto
             ZLIB::ZLIB
             BZip2::BZip2
+            ZSTD::zstd
             PCRE2::pcre2
             LibXml2::LibXml2
             JSONC::jsonc )

libclamav/dconf.c

@@ -109,6 +109,7 @@ static struct dconf_module modules[] = {
     {"ARCHIVE", "UDF", ARCH_CONF_UDF, 1},
     {"ARCHIVE", "LHA", ARCH_CONF_LHA_LZH, 1},
     {"ARCHIVE", "ALZ", ARCH_CONF_ALZ, 1},
+    {"ARCHIVE", "ZSTD", ARCH_CONF_ZSTD, 1},
 
     {"DOCUMENT", "HTML", DOC_CONF_HTML, 1},
     {"DOCUMENT", "RTF", DOC_CONF_RTF, 1},

libclamav/dconf.h

@@ -99,6 +99,7 @@ struct cli_dconf {
 #define ARCH_CONF_UDF     0x8000000
 #define ARCH_CONF_LHA_LZH 0x10000000
 #define ARCH_CONF_ALZ     0x20000000
+#define ARCH_CONF_ZSTD    0x40000000
 
 /* Document flags */
 #define DOC_CONF_HTML         0x1

libclamav/filetypes.c

@@ -143,6 +143,7 @@ static const struct ftmap_s {
     { "CL_TYPE_PYTHON_COMPILED",    CL_TYPE_PYTHON_COMPILED },
     { "CL_TYPE_LHA_LZH",            CL_TYPE_LHA_LZH         },
     { "CL_TYPE_AI_MODEL",           CL_TYPE_AI_MODEL        },
+    { "CL_TYPE_ZSTD",               CL_TYPE_ZSTD            },
     { NULL,                         CL_TYPE_IGNORED         }
 };
 // clang-format on

libclamav/filetypes.h

@@ -96,6 +96,7 @@ typedef enum cli_file {
     CL_TYPE_PYTHON_COMPILED,
     CL_TYPE_LHA_LZH,
     CL_TYPE_AI_MODEL,
+    CL_TYPE_ZSTD,
 
     /* Section for partition types */
     CL_TYPE_PART_ANY, /* unknown partition type */

libclamav/filetypes_int.h

@@ -163,6 +163,7 @@ static const char *ftypes_int[] = {
     "0:0:78617221:XAR container file:CL_TYPE_ANY:CL_TYPE_XAR:75",
     "1:EOF-512:6b6f6c79:DMG container file:CL_TYPE_ANY:CL_TYPE_DMG:75",
     "0:0:fd377a585a00:XZ container file:CL_TYPE_ANY:CL_TYPE_XZ:76",
+    "0:0:28b52ffd:Zstandard compressed file:CL_TYPE_ANY:CL_TYPE_ZSTD:76",
     "4:1024:482B0004:HFS+ partition:CL_TYPE_PART_ANY:CL_TYPE_PART_HFSPLUS:75",
     "4:1024:48580005:HFSX partition:CL_TYPE_PART_ANY:CL_TYPE_PART_HFSPLUS:75",
     "1:0:3c3f786d6c2076657273696f6e3d22312e3022{0-1024}3c576f726b626f6f6b:Microsoft Excel 2003 XML Document:CL_TYPE_ANY:CL_TYPE_XML_XL:80",

libclamav/scanners.c

@@ -52,6 +52,7 @@
 #define DCONF_OTHER ctx->dconf->other
 
 #include <zlib.h>
+#include <zstd.h>
 
 #include "clamav_rust.h"
 #include "clamav.h"
@@ -1420,6 +1421,118 @@ static cl_error_t cli_scanxz(cli_ctx *ctx)
     return ret;
 }
 
+static cl_error_t cli_scanzstd(cli_ctx *ctx)
+{
+    cl_error_t ret = CL_SUCCESS;
+    int fd;
+    char *tmpname;
+    size_t off         = 0;
+    unsigned long size = 0;
+
+    ZSTD_DStream *dstrm = NULL;
+    size_t ores;
+    size_t const obuf_size = ZSTD_DStreamOutSize();
+    size_t const ibuf_size = ZSTD_DStreamInSize();
+    unsigned char *obuf    = NULL;
+
+    cli_dbgmsg("in cli_scanzstd()\n");
+
+    dstrm = ZSTD_createDStream();
+    if (dstrm == NULL) {
+        cli_errmsg("cli_scanzstd: failed to create ZSTD_DStream\n");
+        return CL_EMEM;
+    }
+
+    ores = ZSTD_initDStream(dstrm);
+    if (ZSTD_isError(ores)) {
+        cli_errmsg("cli_scanzstd: ZSTD_initDStream failed: %s\n", ZSTD_getErrorName(ores));
+        ZSTD_freeDStream(dstrm);
+        return CL_EOPEN;
+    }
+
+    obuf = malloc(obuf_size);
+    if (obuf == NULL) {
+        cli_errmsg("cli_scanzstd: no memory for decompress buffer\n");
+        ZSTD_freeDStream(dstrm);
+        return CL_EMEM;
+    }
+
+    if ((ret = cli_gentempfd(ctx->this_layer_tmpdir, &tmpname, &fd))) {
+        cli_errmsg("cli_scanzstd: Can't generate temporary file\n");
+        free(obuf);
+        ZSTD_freeDStream(dstrm);
+        return ret;
+    }
+    cli_dbgmsg("cli_scanzstd: decompressing to file %s\n", tmpname);
+
+    do {
+        size_t avail;
+        ZSTD_inBuffer in;
+        const void *next_in = fmap_need_off_once_len(ctx->fmap, off, ibuf_size, &avail);
+
+        if (!next_in || avail == 0) {
+            break;
+        }
+        off += avail;
+
+        in.src  = next_in;
+        in.size = avail;
+        in.pos  = 0;
+
+        while (in.pos < in.size) {
+            ZSTD_outBuffer out;
+            out.dst  = obuf;
+            out.size = obuf_size;
+            out.pos  = 0;
+
+            ores = ZSTD_decompressStream(dstrm, &out, &in);
+            if (ZSTD_isError(ores)) {
+                cli_dbgmsg("cli_scanzstd: decompress error: %s\n", ZSTD_getErrorName(ores));
+                /* Still scan whatever we've decompressed so far */
+                goto zstd_scan;
+            }
+
+            if (out.pos > 0) {
+                if (cli_writen(fd, obuf, out.pos) != out.pos) {
+                    cli_errmsg("cli_scanzstd: Can't write to file\n");
+                    ret = CL_EWRITE;
+                    goto zstd_exit;
+                }
+                size += out.pos;
+
+                if (cli_checklimits("cli_scanzstd", ctx, size, 0, 0) != CL_SUCCESS) {
+                    cli_warnmsg("cli_scanzstd: decompress file size exceeds limits - "
+                                "only scanning %lu bytes\n",
+                                size);
+                    goto zstd_scan;
+                }
+            }
+
+            if (ores == 0) {
+                /* frame complete; reset to handle concatenated frames */
+                ZSTD_DCtx_reset(dstrm, ZSTD_reset_session_only);
+            }
+        }
+    } while (1);
+
+zstd_scan:
+    if (ret == CL_SUCCESS) {
+        ret = cli_magic_scan_desc(fd, tmpname, ctx, NULL, LAYER_ATTRIBUTES_NONE);
+    }
+
+zstd_exit:
+    ZSTD_freeDStream(dstrm);
+    close(fd);
+    if (!ctx->engine->keeptmp) {
+        if (cli_unlink(tmpname) && ret == CL_SUCCESS) {
+            ret = CL_EUNLINK;
+        }
+    }
+    free(tmpname);
+    free(obuf);
+    return ret;
+}
+
 static cl_error_t cli_scanszdd(cli_ctx *ctx)
 {
     int ofd;
@@ -4854,6 +4967,11 @@ cl_error_t cli_magic_scan(cli_ctx *ctx, cli_file_t type)
                 ret = cli_scanxz(ctx);
             break;
 
+        case CL_TYPE_ZSTD:
+            if (SCAN_PARSE_ARCHIVE && (DCONF_ARCH & ARCH_CONF_ZSTD))
+                ret = cli_scanzstd(ctx);
+            break;
+
         case CL_TYPE_GPT:
             if (SCAN_PARSE_ARCHIVE && (DCONF_ARCH & ARCH_CONF_GPT))
                 ret = cli_scangpt(ctx, 0);