Add Zstandard compression support and update tests

CMakeLists.txt

@@ -341,6 +341,33 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR})
 set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS})
 find_package(ZLIB REQUIRED)
 
+find_package(zstd CONFIG QUIET)
+if(zstd_FOUND)
+
+  # Provide a compatibility target name if the upstream package does not export it
+  # Our code links against `zstd::zstd`; upstream zstd usually exports
+  # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed.
+  if(NOT TARGET zstd::zstd)
+    if(TARGET zstd::libzstd_shared)
+      set(_zstd_target zstd::libzstd_shared)
+    elseif(TARGET zstd::libzstd_static)
+      set(_zstd_target zstd::libzstd_static)
+    elseif(TARGET zstd::libzstd)
+      set(_zstd_target zstd::libzstd)
+    endif()
+    if(DEFINED _zstd_target)
+      add_library(zstd_zstd INTERFACE)
+      target_link_libraries(zstd_zstd INTERFACE ${_zstd_target})
+      add_library(zstd::zstd ALIAS zstd_zstd)
+      set(HAVE_ZSTD_H TRUE)
+    else()
+      set(HAVE_ZSTD_H FALSE)
+    endif()
+  endif()
+else()
+  set(HAVE_ZSTD_H FALSE)
+endif()
+
 # ncurses is used in traffic_top
 find_package(Curses)
 set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H})

ci/docker/deb/Dockerfile

@@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \
     apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \
     libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \
     tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \
-    libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \
+    libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \
+    libzstd-dev; \
     # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed)
     apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \
     groupadd  -g 665 jenkins && \

ci/docker/yum/Dockerfile

@@ -52,7 +52,7 @@ RUN yum -y update; \
     # Devel packages that ATS needs
     yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \
     xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \
-    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \
+    ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \
     perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \
     # This is for autest stuff
     yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \

contrib/docker/ubuntu/noble/Dockerfile

@@ -48,6 +48,7 @@ RUN apt update \
     libpcre3-dev \
     hwloc \
     libbrotli-dev \
+    libzstd-dev \
     luajit \
     libcap-dev \
     libmagick++-dev \

doc/admin-guide/files/records.yaml.en.rst

@@ -2076,10 +2076,14 @@ Proxy User Variables
          normalize as for value ``1``
    ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE**
          normalize as for value ``2``
+   ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE**
+         normalize as for value ``2``
+   ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE**
+         normalize as for value ``4``
    ===== ======================================================================
 
    This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``).
-   Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli).
+   Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard).
 
 Security
 ========

doc/admin-guide/plugins/compress.en.rst

@@ -192,12 +192,59 @@ supported-algorithms
 
 Provides the compression algorithms that are supported, a comma separate list
 of values. This will allow |TS| to selectively support ``gzip``, ``deflate``,
-and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can
-be selected using ',' delimiter, for instance, ``supported-algorithms
-deflate,gzip,br``. Note that this list must **not** contain any white-spaces!
+brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``.
+Multiple algorithms can be selected using ',' delimiter, for instance,
+``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not**
+contain any white-spaces!
+
+============== =================================================================
+Algorithm      Description
+============== =================================================================
+gzip           Standard gzip compression (default, widely supported)
+deflate        Deflate compression (RFC 1951)
+br             Brotli compression (modern, efficient)
+zstd           Zstandard compression (fast, high compression ratio)
+============== =================================================================
 
 Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will
-be considered, and if it is ``2``, only br or gzip will be considered.
+be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``,
+only zstd, br, or gzip will be considered, and if it is ``5``, all combinations
+of zstd, br, and gzip will be considered.
+
+gzip-compression-level
+-----------------------
+
+Sets the compression level for gzip compression. Valid values are 1-9, where
+1 is fastest compression (lowest compression ratio) and 9 is slowest compression
+(highest compression ratio). The default is 6, which provides a good balance
+between compression speed and ratio.
+
+brotli-compression-level
+-------------------------
+
+Sets the compression level for Brotli compression. Valid values are 0-11, where
+0 is fastest compression (lowest compression ratio) and 11 is slowest compression
+(highest compression ratio). The default is 6, which provides a good balance
+between compression speed and ratio.
+
+brotli-lgwin
+------------
+
+Sets the window size for Brotli compression. Valid values are 10-24, where
+larger values provide better compression but use more memory. The default is 16.
+This parameter controls the sliding window size used during compression:
+
+- 10: 1KB window (fastest, least memory)
+- 16: 64KB window (default, good balance)
+- 24: 16MB window (slowest, most memory, best compression)
+
+zstd-compression-level
+----------------------
+
+Sets the compression level for Zstandard compression. Valid values are 1-22, where
+1 is fastest compression (lowest compression ratio) and 22 is slowest compression
+(highest compression ratio). The default is 12, which provides an excellent
+balance between compression speed and ratio for web content.
 
 Examples
 ========
@@ -214,6 +261,10 @@ might create a configuration with the following options::
    compressible-status-code 200, 206
    minimum-content-length 860
    flush false
+   gzip-compression-level 6
+   brotli-compression-level 6
+   brotli-lgwin 16
+   zstd-compression-level 12
 
    # Now set a configuration for www.example.com
    [www.example.com]
@@ -231,13 +282,37 @@ might create a configuration with the following options::
    flush true
    supported-algorithms gzip,deflate
 
-   # Supports brotli compression
+   # Supports brotli compression with custom settings
    [brotli.compress.com]
    enabled true
    compressible-content-type text/*
    compressible-content-type application/json
    flush true
    supported-algorithms br,gzip
+   brotli-compression-level 8
+   brotli-lgwin 20
+
+   # Supports zstd compression for high efficiency
+   [zstd.compress.com]
+   enabled true
+   compressible-content-type text/*
+   compressible-content-type application/json
+   compressible-content-type application/javascript
+   flush true
+   supported-algorithms zstd,gzip
+   zstd-compression-level 15
+
+   # Supports all compression algorithms with optimized settings
+   [all.compress.com]
+   enabled true
+   compressible-content-type text/*
+   compressible-content-type application/json
+   flush true
+   supported-algorithms zstd,br,gzip,deflate
+   gzip-compression-level 7
+   brotli-compression-level 9
+   brotli-lgwin 18
+   zstd-compression-level 10
 
    # This origin does it all
    [bar.example.com]

doc/developer-guide/plugins/http-headers/header-functions.en.rst

@@ -92,6 +92,9 @@ headers.
 ``TS_HTTP_VALUE_GZIP``
    "gzip"
 
+``TS_HTTP_VALUE_ZSTD``
+   "zstd"
+
 ``TS_HTTP_VALUE_IDENTITY``
    "identity"
 

doc/release-notes/whats-new.en.rst

@@ -81,6 +81,7 @@ Plugins
 * xdebug - ``--enable`` option to selectively enable features has been added
 * system_stats - Stats about memory have been added
 * slice plugin - This plugin was promoted to stable.
+* compress plugin - Added support for Zstandard (zstd) compression algorithm.
 
 JSON-RPC
 ^^^^^^^^

include/proxy/hdrs/HTTP.h

@@ -368,6 +368,7 @@ extern c_str_view HTTP_VALUE_COMPRESS;
 extern c_str_view HTTP_VALUE_DEFLATE;
 extern c_str_view HTTP_VALUE_GZIP;
 extern c_str_view HTTP_VALUE_BROTLI;
+extern c_str_view HTTP_VALUE_ZSTD;
 extern c_str_view HTTP_VALUE_IDENTITY;
 extern c_str_view HTTP_VALUE_KEEP_ALIVE;
 extern c_str_view HTTP_VALUE_MAX_AGE;

include/proxy/hdrs/MIME.h

@@ -602,6 +602,8 @@ extern c_str_view MIME_VALUE_COMPRESS;
 extern c_str_view MIME_VALUE_DEFLATE;
 extern c_str_view MIME_VALUE_GZIP;
 extern c_str_view MIME_VALUE_BROTLI;
+extern c_str_view MIME_VALUE_ZSTD;
+
 extern c_str_view MIME_VALUE_IDENTITY;
 extern c_str_view MIME_VALUE_KEEP_ALIVE;
 extern c_str_view MIME_VALUE_MAX_AGE;

include/ts/apidefs.h.in

@@ -1351,6 +1351,7 @@ extern const char *TS_HTTP_VALUE_COMPRESS;
 extern const char *TS_HTTP_VALUE_DEFLATE;
 extern const char *TS_HTTP_VALUE_GZIP;
 extern const char *TS_HTTP_VALUE_BROTLI;
+extern const char *TS_HTTP_VALUE_ZSTD;
 extern const char *TS_HTTP_VALUE_IDENTITY;
 extern const char *TS_HTTP_VALUE_KEEP_ALIVE;
 extern const char *TS_HTTP_VALUE_MAX_AGE;
@@ -1375,6 +1376,7 @@ extern int TS_HTTP_LEN_COMPRESS;
 extern int TS_HTTP_LEN_DEFLATE;
 extern int TS_HTTP_LEN_GZIP;
 extern int TS_HTTP_LEN_BROTLI;
+extern int TS_HTTP_LEN_ZSTD;
 extern int TS_HTTP_LEN_IDENTITY;
 extern int TS_HTTP_LEN_KEEP_ALIVE;
 extern int TS_HTTP_LEN_MAX_AGE;

include/tscore/ink_config.h.cmake.in

@@ -46,6 +46,7 @@
 #cmakedefine HAVE_POSIX_FADVISE 1
 #cmakedefine HAVE_POSIX_FALLOCATE 1
 #cmakedefine HAVE_POSIX_MADVISE 1
+#cmakedefine HAVE_ZSTD_H 1
 
 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1
 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1

plugins/compress/CMakeLists.txt

@@ -20,5 +20,10 @@ target_link_libraries(compress PRIVATE libswoc::libswoc)
 if(HAVE_BROTLI_ENCODE_H)
   target_link_libraries(compress PRIVATE brotli::brotlienc)
 endif()
+
+if(HAVE_ZSTD_H)
+  target_link_libraries(compress PRIVATE zstd::zstd)
+endif()
+
 verify_global_plugin(compress)
 verify_remap_plugin(compress)

plugins/compress/README

@@ -1,7 +1,7 @@
 What this plugin does:
 =====================
 
-This plugin compresses responses, via gzip or brotli, whichever is applicable
+This plugin compresses responses, via gzip, deflate, brotli, or zstd (Zstandard), whichever is applicable
 it can compress origin responses as well as cached responses
 
 installation:
@@ -24,4 +24,4 @@ compress.so <path-to-config>/sample.compress.config
 After modifying plugin.config, restart traffic server (sudo traffic_ctl server restart)
 the configuration is re-read when a management update is given (sudo traffic_ctl config reload)
 
-See sample.config.compress for an example configuration and the options that are available
+See sample.compress.config for an example configuration and the options that are available