From b22cbee7c65e89d4e5dacd62d1821658cada1414 Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 01:30:58 +0100 Subject: [PATCH 01/15] Add comprehensive zstd compression support to ATS This patch adds full support for the zstd (Zstandard) compression algorithm throughout Apache Traffic Server, including build system integration, compression plugin support, Accept-Encoding header normalization, and comprehensive test coverage. Build system and dependencies: - Add CMake support for finding zstd library with new Findzstd.cmake - Update Docker build files to include libzstd-dev package - Add TS_HAS_ZSTD feature flag for conditional compilation Core compression support: - Extend compress plugin to support zstd compression alongside gzip and brotli - Add zstd stream handling structures and functions - Update compression configuration to include zstd in supported algorithms list - Add zstd compression type constant and related infrastructure Accept-Encoding header normalization: - Extend proxy.config.http.normalize_ae configuration to support values 4 and 5 for zstd normalization - Add zstd support to header normalization logic with proper priority handling (zstd > br > gzip) - Update HTTP transaction cache matching to handle zstd encoding - Add zstd token to header parsing infrastructure API and infrastructure: - Add TS_HTTP_VALUE_ZSTD and TS_HTTP_LEN_ZSTD constants - Update MIME field handling to recognize zstd encoding - Add zstd support to traffic_layout feature detection Test coverage: - Expand compress plugin tests to cover zstd compression scenarios - Add zstd test cases to Accept-Encoding normalization tests - Update golden files to include zstd compression test results - Add new compress3.config for zstd-specific plugin configuration - Test all combinations of zstd, br, and gzip in various scenarios The implementation follows RFC 8878 standards for zstd compression and maintains backward compatibility with existing gzip and brotli compression functionality. All tests pass and the feature is properly integrated with the existing caching and content negotiation mechanisms. --- CMakeLists.txt | 5 + ci/docker/deb/Dockerfile | 3 +- ci/docker/yum/Dockerfile | 2 +- cmake/Findzstd.cmake | 53 ++ contrib/docker/ubuntu/noble/Dockerfile | 1 + doc/admin-guide/files/records.yaml.en.rst | 6 +- doc/admin-guide/plugins/compress.en.rst | 37 +- .../http-headers/header-functions.en.rst | 3 + doc/release-notes/whats-new.en.rst | 1 + include/proxy/hdrs/HTTP.h | 1 + include/proxy/hdrs/MIME.h | 2 + include/ts/apidefs.h.in | 2 + include/tscore/ink_config.h.cmake.in | 2 + plugins/compress/CMakeLists.txt | 5 + plugins/compress/README | 2 +- plugins/compress/compress.cc | 183 +++- plugins/compress/configuration.cc | 8 +- plugins/compress/configuration.h | 3 +- plugins/compress/misc.cc | 11 +- plugins/compress/misc.h | 22 +- plugins/compress/sample.compress.config | 2 +- src/api/InkAPIInternal.cc | 3 + src/proxy/hdrs/HTTP.cc | 2 + src/proxy/hdrs/HdrToken.cc | 5 +- src/proxy/hdrs/MIME.cc | 2 + src/proxy/http/HttpTransactHeaders.cc | 47 + src/records/RecordsConfig.cc | 2 +- src/traffic_layout/info.cc | 5 + tests/gold_tests/headers/normalize_ae.gold | 332 +++++++ tests/gold_tests/headers/normalize_ae.test.py | 53 ++ .../normalized_ae_match_vary_cache.test.py | 6 + ...malized_ae_varied_transactions.replay.yaml | 830 ++++++++++++++++++ .../pluginTest/compress/compress.gold | 224 ++++- .../pluginTest/compress/compress.test.py | 72 +- .../pluginTest/compress/compress3.config | 7 + .../pluginTest/compress/compress_userver.gold | 24 +- 36 files changed, 1881 insertions(+), 87 deletions(-) create mode 100644 cmake/Findzstd.cmake create mode 100644 tests/gold_tests/pluginTest/compress/compress3.config diff --git a/CMakeLists.txt b/CMakeLists.txt index 010d74fc708..df0f61910f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,11 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) +find_package(ZSTD) +if(ZSTD_FOUND) + set(HAVE_ZSTD_H TRUE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 337356ca8c3..4e1398d15b7 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \ apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ - libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \ + libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ + libzstd-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 85e9a7add64..5a160fb24ff 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake new file mode 100644 index 00000000000..d2a86d0132e --- /dev/null +++ b/cmake/Findzstd.cmake @@ -0,0 +1,53 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# Findzstd.cmake +# +# This will define the following variables +# +# ZSTD_FOUND +# ZSTD_LIBRARY +# ZSTD_INCLUDE_DIRS +# +# and the following imported target +# +# zstd::zstd +# + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) + +mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +include(SelectLibraryConfigurations) +select_library_configurations(ZSTD) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") +endif() + +if(ZSTD_FOUND AND NOT TARGET zstd::zstd) + add_library(zstd::zstd INTERFACE IMPORTED) + target_include_directories(zstd::zstd INTERFACE ${ZSTD_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${ZSTD_LIBRARY}") +endif() diff --git a/contrib/docker/ubuntu/noble/Dockerfile b/contrib/docker/ubuntu/noble/Dockerfile index 3325e86da4c..b9181dc998f 100644 --- a/contrib/docker/ubuntu/noble/Dockerfile +++ b/contrib/docker/ubuntu/noble/Dockerfile @@ -48,6 +48,7 @@ RUN apt update \ libpcre3-dev \ hwloc \ libbrotli-dev \ + libzstd-dev \ luajit \ libcap-dev \ libmagick++-dev \ diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index ed5f987388f..bf8e8ae5fa2 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2076,10 +2076,14 @@ Proxy User Variables normalize as for value ``1`` ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE** normalize as for value ``2`` + ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE** + normalize as for value ``2`` + ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE** + normalize as for value ``4`` ===== ====================================================================== This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``). - Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli). + Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard). Security ======== diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 81dec81dc8f..686445008fc 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -192,12 +192,24 @@ supported-algorithms Provides the compression algorithms that are supported, a comma separate list of values. This will allow |TS| to selectively support ``gzip``, ``deflate``, -and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can -be selected using ',' delimiter, for instance, ``supported-algorithms -deflate,gzip,br``. Note that this list must **not** contain any white-spaces! +brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``. +Multiple algorithms can be selected using ',' delimiter, for instance, +``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not** +contain any white-spaces! + +============== ================================================================= +Algorithm Description +============== ================================================================= +gzip Standard gzip compression (default, widely supported) +deflate Deflate compression (RFC 1951) +br Brotli compression (modern, efficient) +zstd Zstandard compression (fast, high compression ratio) +============== ================================================================= Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will -be considered, and if it is ``2``, only br or gzip will be considered. +be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``, +only zstd, br, or gzip will be considered, and if it is ``5``, all combinations +of zstd, br, and gzip will be considered. Examples ======== @@ -239,6 +251,23 @@ might create a configuration with the following options:: flush true supported-algorithms br,gzip + # Supports zstd compression for high efficiency + [zstd.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + compressible-content-type application/javascript + flush true + supported-algorithms zstd,gzip + + # Supports all compression algorithms + [all.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + flush true + supported-algorithms zstd,br,gzip,deflate + # This origin does it all [bar.example.com] enabled false diff --git a/doc/developer-guide/plugins/http-headers/header-functions.en.rst b/doc/developer-guide/plugins/http-headers/header-functions.en.rst index 9a27115434f..47d07459a2c 100644 --- a/doc/developer-guide/plugins/http-headers/header-functions.en.rst +++ b/doc/developer-guide/plugins/http-headers/header-functions.en.rst @@ -92,6 +92,9 @@ headers. ``TS_HTTP_VALUE_GZIP`` "gzip" +``TS_HTTP_VALUE_ZSTD`` + "zstd" + ``TS_HTTP_VALUE_IDENTITY`` "identity" diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst index c2a41318129..ac52126e24c 100644 --- a/doc/release-notes/whats-new.en.rst +++ b/doc/release-notes/whats-new.en.rst @@ -81,6 +81,7 @@ Plugins * xdebug - ``--enable`` option to selectively enable features has been added * system_stats - Stats about memory have been added * slice plugin - This plugin was promoted to stable. +* compress plugin - Added support for Zstandard (zstd) compression algorithm. JSON-RPC ^^^^^^^^ diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index eeecba31d74..8759fcc09ab 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h @@ -368,6 +368,7 @@ extern c_str_view HTTP_VALUE_COMPRESS; extern c_str_view HTTP_VALUE_DEFLATE; extern c_str_view HTTP_VALUE_GZIP; extern c_str_view HTTP_VALUE_BROTLI; +extern c_str_view HTTP_VALUE_ZSTD; extern c_str_view HTTP_VALUE_IDENTITY; extern c_str_view HTTP_VALUE_KEEP_ALIVE; extern c_str_view HTTP_VALUE_MAX_AGE; diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index dcea4893594..4bed80dc13c 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -602,6 +602,8 @@ extern c_str_view MIME_VALUE_COMPRESS; extern c_str_view MIME_VALUE_DEFLATE; extern c_str_view MIME_VALUE_GZIP; extern c_str_view MIME_VALUE_BROTLI; +extern c_str_view MIME_VALUE_ZSTD; + extern c_str_view MIME_VALUE_IDENTITY; extern c_str_view MIME_VALUE_KEEP_ALIVE; extern c_str_view MIME_VALUE_MAX_AGE; diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index bd37127970d..05049a6d95b 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -1351,6 +1351,7 @@ extern const char *TS_HTTP_VALUE_COMPRESS; extern const char *TS_HTTP_VALUE_DEFLATE; extern const char *TS_HTTP_VALUE_GZIP; extern const char *TS_HTTP_VALUE_BROTLI; +extern const char *TS_HTTP_VALUE_ZSTD; extern const char *TS_HTTP_VALUE_IDENTITY; extern const char *TS_HTTP_VALUE_KEEP_ALIVE; extern const char *TS_HTTP_VALUE_MAX_AGE; @@ -1375,6 +1376,7 @@ extern int TS_HTTP_LEN_COMPRESS; extern int TS_HTTP_LEN_DEFLATE; extern int TS_HTTP_LEN_GZIP; extern int TS_HTTP_LEN_BROTLI; +extern int TS_HTTP_LEN_ZSTD; extern int TS_HTTP_LEN_IDENTITY; extern int TS_HTTP_LEN_KEEP_ALIVE; extern int TS_HTTP_LEN_MAX_AGE; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index fcbd98a5822..2d515338c8a 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -187,3 +187,5 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine YAMLCPP_LIB_VERSION "@YAMLCPP_LIB_VERSION@" #cmakedefine01 TS_HAS_CRIPTS + +#cmakedefine HAVE_ZSTD_H 1 diff --git a/plugins/compress/CMakeLists.txt b/plugins/compress/CMakeLists.txt index f630bf0d1c9..65b24ce9558 100644 --- a/plugins/compress/CMakeLists.txt +++ b/plugins/compress/CMakeLists.txt @@ -20,5 +20,10 @@ target_link_libraries(compress PRIVATE libswoc::libswoc) if(HAVE_BROTLI_ENCODE_H) target_link_libraries(compress PRIVATE brotli::brotlienc) endif() + +if(HAVE_ZSTD_H) + target_link_libraries(compress PRIVATE zstd::zstd) +endif() + verify_global_plugin(compress) verify_remap_plugin(compress) diff --git a/plugins/compress/README b/plugins/compress/README index 759add28e03..e89070f9e54 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -1,7 +1,7 @@ What this plugin does: ===================== -This plugin compresses responses, via gzip or brotli, whichever is applicable +This plugin compresses responses, via gzip, deflate, brotli, or zstd (Zstandard), whichever is applicable it can compress origin responses as well as cached responses installation: diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 44e8235a0d0..b181d675722 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -23,6 +23,9 @@ #include #include +#if HAVE_ZSTD_H +#include +#endif #include "ts/apidefs.h" #include "tscore/ink_config.h" @@ -71,6 +74,10 @@ const int BROTLI_COMPRESSION_LEVEL = 6; const int BROTLI_LGW = 16; #endif +#if HAVE_ZSTD_H +const int ZSTD_COMPRESSION_LEVEL = 6; +#endif + static const char *global_hidden_header_name = nullptr; static TSMutex compress_config_mutex = nullptr; @@ -136,6 +143,13 @@ handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) } } // namespace +// Forward declarations for ZSTD compression functions +#if HAVE_ZSTD_H +static void zstd_compress_init(Data *data); +static void zstd_compress_finish(Data *data); +static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length); +#endif + static Data * data_alloc(int compression_type, int compression_algorithms) { @@ -195,6 +209,22 @@ data_alloc(int compression_type, int compression_algorithms) data->bstrm.avail_out = 0; data->bstrm.total_out = 0; } +#endif +#if HAVE_ZSTD_H + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.next_in = nullptr; + data->zstrm_zstd.avail_in = 0; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.next_out = nullptr; + data->zstrm_zstd.avail_out = 0; + data->zstrm_zstd.total_out = 0; + if (compression_type & COMPRESSION_TYPE_ZSTD) { + debug("zstd compression. Create Zstd Compression Context."); + data->zstrm_zstd.cctx = ZSTD_createCCtx(); + if (!data->zstrm_zstd.cctx) { + fatal("Zstd Compression Context Creation Failed"); + } + } #endif return data; } @@ -216,6 +246,11 @@ data_destroy(Data *data) #if HAVE_BROTLI_ENCODE_H BrotliEncoderDestroyInstance(data->bstrm.br); #endif +#if HAVE_ZSTD_H + if (data->zstrm_zstd.cctx) { + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + } +#endif TSfree(data); } @@ -228,7 +263,10 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty const char *value = nullptr; int value_len = 0; // Delete Content-Encoding if present??? - if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { + if (compression_type & COMPRESSION_TYPE_ZSTD && (algorithm & ALGORITHM_ZSTD)) { + value = TS_HTTP_VALUE_ZSTD; + value_len = TS_HTTP_LEN_ZSTD; + } else if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { value = TS_HTTP_VALUE_BROTLI; value_len = TS_HTTP_LEN_BROTLI; } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP)) { @@ -240,7 +278,6 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty } if (value_len == 0) { - error("no need to add Content-Encoding header"); return TS_SUCCESS; } @@ -361,6 +398,16 @@ compress_transform_init(TSCont contp, Data *data) data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX); } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD) { + zstd_compress_init(data); + if (!data->zstrm_zstd.cctx) { + TSError("Failed to create Zstandard compression context"); + return; + } + } +#endif + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); } @@ -466,6 +513,112 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l } #endif +#if HAVE_ZSTD_H +static void +zstd_compress_init(Data *data) +{ + if (!data->zstrm_zstd.cctx) { + error("Failed to initialize Zstd compression context"); + return; + } + + // Set compression level + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL); + if (ZSTD_isError(result)) { + error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); + return; + } + + // Enable checksum for data integrity + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); + return; + } + + debug("zstd compression context initialized with level %d", ZSTD_COMPRESSION_LEVEL); +} + +static void +zstd_compress_finish(Data *data) +{ + if (data->state == transform_state_output) { + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + // Finalize the zstd stream + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); + + if (ZSTD_isError(remaining)) { + error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); + break; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (remaining == 0) { /* compression finished */ + break; + } + } + + debug("zstd-transform: Finished zstd compression"); + log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); + } +} + +static void +zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + // Set up input buffer for zstd streaming + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + data->zstrm_zstd.total_in += upstream_length; + + while (input.pos < input.size) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + // Set up output buffer for zstd streaming + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + // Compress the data using streaming API + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, ZSTD_e_continue); + + if (ZSTD_isError(result)) { + error("Zstd compression failed: %s", ZSTD_getErrorName(result)); + return; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + // If we have output space but no more input was consumed, break to avoid infinite loop + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + break; + } + } +} +#endif + static void compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) { @@ -488,8 +641,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) upstream_length = amount; } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + zstd_compress_one(data, upstream_buffer, upstream_length); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { brotli_transform_one(data, upstream_buffer, upstream_length); } else #endif @@ -575,8 +733,14 @@ brotli_transform_finish(Data *data) static void compress_transform_finish(Data *data) { +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + zstd_compress_finish(data); + debug("compress_transform_finish: zstd compression finish"); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { brotli_transform_finish(data); debug("compress_transform_finish: brotli compression finish"); } else @@ -787,7 +951,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration continue; } - if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { + info("Accept-Encoding value [%.*s]", len, value); + + if (strncasecmp(value, "zstd", sizeof("zstd") - 1) == 0) { + if (*algorithms & ALGORITHM_ZSTD) { + compression_acceptable = 1; + } + *compress_type |= COMPRESSION_TYPE_ZSTD; + } else if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { if (*algorithms & ALGORITHM_BROTLI) { compression_acceptable = 1; } diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index dd95952f97e..41b35964eac 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -229,6 +229,12 @@ HostConfiguration::add_compression_algorithms(string &line) string token = extractFirstToken(line, isCommaOrSpace); if (token.empty()) { break; + } else if (token == "zstd") { +#ifdef HAVE_ZSTD_H + compression_algorithms_ |= ALGORITHM_ZSTD; +#else + error("supported-algorithms: zstd support not compiled in."); +#endif } else if (token == "br") { #ifdef HAVE_BROTLI_ENCODE_H compression_algorithms_ |= ALGORITHM_BROTLI; @@ -240,7 +246,7 @@ HostConfiguration::add_compression_algorithms(string &line) } else if (token == "deflate") { compression_algorithms_ |= ALGORITHM_DEFLATE; } else { - error("Unknown compression type. Supported compression-algorithms ."); + error("Unknown compression type. Supported compression-algorithms ."); } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 22b2173c590..2140d5a3580 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -38,7 +38,8 @@ enum CompressionAlgorithm { ALGORITHM_DEFAULT = 0, ALGORITHM_DEFLATE = 1, ALGORITHM_GZIP = 2, - ALGORITHM_BROTLI = 4 // For bit manipulations + ALGORITHM_BROTLI = 4, + ALGORITHM_ZSTD = 8 }; enum class RangeRequestCtrl : int { diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index a1b33b79cce..13d47e7aee0 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -84,8 +84,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo bool deflate = false; bool gzip = false; bool br = false; + bool zstd = false; // remove the accept encoding field(s), - // while finding out if gzip or deflate is supported. + // while finding out if gzip, brotli, deflate, or zstandard are supported. while (field) { int val_len; const char *values_ = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, -1, &val_len); @@ -100,6 +101,8 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo br = true; } else if (strcasecmp("deflate", next) == 0) { deflate = true; + } else if (strcasecmp("zstd", next) == 0) { + zstd = true; } } } @@ -111,9 +114,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo } // append a new accept-encoding field in the header - if (deflate || gzip || br) { + if (deflate || gzip || br || zstd) { TSMimeHdrFieldCreate(reqp, hdr_loc, &field); TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + if (zstd) { + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "zstd", strlen("zstd")); + info("normalized accept encoding to zstd"); + } if (br) { TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br")); info("normalized accept encoding to br"); diff --git a/plugins/compress/misc.h b/plugins/compress/misc.h index 9491271df36..4a3da5d99a5 100644 --- a/plugins/compress/misc.h +++ b/plugins/compress/misc.h @@ -32,6 +32,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + #include "configuration.h" // zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html @@ -44,7 +48,8 @@ enum CompressionType { COMPRESSION_TYPE_DEFAULT = 0, COMPRESSION_TYPE_DEFLATE = 1, COMPRESSION_TYPE_GZIP = 2, - COMPRESSION_TYPE_BROTLI = 4 + COMPRESSION_TYPE_BROTLI = 4, + COMPRESSION_TYPE_ZSTD = 8, }; // this one is used to rename the accept encoding header @@ -70,6 +75,18 @@ using b_stream = struct { }; #endif +#if HAVE_ZSTD_H +using zstd_stream = struct { + ZSTD_CCtx *cctx; + const void *next_in; + size_t avail_in; + void *next_out; + size_t avail_out; + size_t total_in; + size_t total_out; +}; +#endif + using Data = struct { TSHttpTxn txn; Gzip::HostConfiguration *hc; @@ -84,6 +101,9 @@ using Data = struct { #if HAVE_BROTLI_ENCODE_H b_stream bstrm; #endif +#if HAVE_ZSTD_H + zstd_stream zstrm_zstd; +#endif }; voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index b1431a97794..8b0eaaf5be8 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -56,7 +56,7 @@ allow !*/bla* minimum-content-length 1024 #supported algorithms -supported-algorithms br,gzip +supported-algorithms br,gzip,zstd #override the global configuration for a host. #www.foo.nl does NOT inherit anything diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index 85f00287c52..9a4bb620ab1 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -232,6 +232,7 @@ const char *TS_HTTP_VALUE_COMPRESS; const char *TS_HTTP_VALUE_DEFLATE; const char *TS_HTTP_VALUE_GZIP; const char *TS_HTTP_VALUE_BROTLI; +const char *TS_HTTP_VALUE_ZSTD; const char *TS_HTTP_VALUE_IDENTITY; const char *TS_HTTP_VALUE_KEEP_ALIVE; const char *TS_HTTP_VALUE_MAX_AGE; @@ -256,6 +257,7 @@ int TS_HTTP_LEN_COMPRESS; int TS_HTTP_LEN_DEFLATE; int TS_HTTP_LEN_GZIP; int TS_HTTP_LEN_BROTLI; +int TS_HTTP_LEN_ZSTD; int TS_HTTP_LEN_IDENTITY; int TS_HTTP_LEN_KEEP_ALIVE; int TS_HTTP_LEN_MAX_AGE; @@ -748,6 +750,7 @@ api_init() TS_HTTP_VALUE_DEFLATE = HTTP_VALUE_DEFLATE.c_str(); TS_HTTP_VALUE_GZIP = HTTP_VALUE_GZIP.c_str(); TS_HTTP_VALUE_BROTLI = HTTP_VALUE_BROTLI.c_str(); + TS_HTTP_VALUE_ZSTD = HTTP_VALUE_ZSTD.c_str(); TS_HTTP_VALUE_IDENTITY = HTTP_VALUE_IDENTITY.c_str(); TS_HTTP_VALUE_KEEP_ALIVE = HTTP_VALUE_KEEP_ALIVE.c_str(); TS_HTTP_VALUE_MAX_AGE = HTTP_VALUE_MAX_AGE.c_str(); diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index a6c41c5f4a3..5f80ceab54f 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -78,6 +78,7 @@ c_str_view HTTP_VALUE_COMPRESS; c_str_view HTTP_VALUE_DEFLATE; c_str_view HTTP_VALUE_GZIP; c_str_view HTTP_VALUE_BROTLI; +c_str_view HTTP_VALUE_ZSTD; c_str_view HTTP_VALUE_IDENTITY; c_str_view HTTP_VALUE_KEEP_ALIVE; c_str_view HTTP_VALUE_MAX_AGE; @@ -183,6 +184,7 @@ http_init() HTTP_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); HTTP_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); HTTP_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + HTTP_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); HTTP_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); HTTP_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); HTTP_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index 3fde664e1c3..afb81e82ad9 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -122,7 +122,10 @@ const char *const _hdrtoken_strs[] = { "Early-Data", // RFC-7932 - "br"}; + "br", + + // RFC-8878 + "zstd"}; HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HdrTokenType::SCHEME }, diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index f46fcdbd317..6a6234fdd86 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -164,6 +164,7 @@ c_str_view MIME_VALUE_COMPRESS; c_str_view MIME_VALUE_DEFLATE; c_str_view MIME_VALUE_GZIP; c_str_view MIME_VALUE_BROTLI; +c_str_view MIME_VALUE_ZSTD; c_str_view MIME_VALUE_IDENTITY; c_str_view MIME_VALUE_KEEP_ALIVE; c_str_view MIME_VALUE_MAX_AGE; @@ -766,6 +767,7 @@ mime_init() MIME_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); MIME_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); MIME_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + MIME_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); MIME_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); MIME_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); MIME_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index 4a26790675e..a7ecee2eb32 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc @@ -1241,6 +1241,53 @@ HttpTransactHeaders::normalize_accept_encoding(const OverridableHttpConfigParams header->field_delete(ae_field); Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-br non-gzip Accept-Encoding"); } + } else if (normalize_ae == 4) { + // Force Accept-Encoding header to zstd or fallback to br/gzip or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } + } else if (normalize_ae == 5) { + // Force Accept-Encoding header to zstd,br,gzip combinations or individual algorithms or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "zstd, br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } } else { static bool logged = false; diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 2772f23fc49..554eb951a65 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -535,7 +535,7 @@ static const RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.http.allow_multi_range", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , // This defaults to a special invalid value so the HTTP transaction handling code can tell that it was not explicitly set. - {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-5]", RECA_NULL} , // #################################################### diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 62732e09473..541985112cc 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -91,6 +91,11 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif +#if HAVE_ZSTD_H + print_feature("TS_HAS_ZSTD", 1, json); +#else + print_feature("TS_HAS_ZSTD", 0, json); +#endif #ifdef F_GETPIPE_SZ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json); #else diff --git a/tests/gold_tests/headers/normalize_ae.gold b/tests/gold_tests/headers/normalize_ae.gold index 5e44f966741..f9b2c675ae9 100644 --- a/tests/gold_tests/headers/normalize_ae.gold +++ b/tests/gold_tests/headers/normalize_ae.gold @@ -11,6 +11,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -24,6 +44,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -37,6 +77,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -50,6 +110,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -63,6 +143,92 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- X-Au-Test: www.no-oride.com ACCEPT-ENCODING MISSING - @@ -76,6 +242,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -89,6 +275,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -102,6 +308,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -115,6 +341,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -128,3 +374,89 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- diff --git a/tests/gold_tests/headers/normalize_ae.test.py b/tests/gold_tests/headers/normalize_ae.test.py index 313023150d6..ca38755bfe9 100644 --- a/tests/gold_tests/headers/normalize_ae.test.py +++ b/tests/gold_tests/headers/normalize_ae.test.py @@ -40,6 +40,10 @@ server.addResponse("sessionlog.json", request_header, response_header) request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-2.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-4.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-5.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) # Define first ATS. Disable the cache to make sure each request is sent to the # origin server. @@ -65,6 +69,12 @@ def baselineTsSetup(ts): ts.Disk.remap_config.AddLine( 'map http://www.ae-3.com http://127.0.0.1:{0}'.format(server.Variables.Port) + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-4.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-5.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') baselineTsSetup(ts) @@ -124,6 +134,47 @@ def curlTail(hdrValue): tr.MakeCurlCommand(baseCurl + curlTail('gzip;q=0.3, whatever;q=0.666, br;q=0.7'), ts=ts) tr.Processes.Default.ReturnCode = 0 + # ZSTD-related tests for normalize_ae modes 4 and 5 + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('gzip, zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd;q=0.8, br;q=0.7, gzip;q=0.6')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('deflate, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('identity, zstd, compress')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, compress')) + tr.Processes.Default.ReturnCode = 0 + def perTsTest(shouldWaitForUServer, ts): allAEHdrs(shouldWaitForUServer, True, ts, 'www.no-oride.com') @@ -131,6 +182,8 @@ def perTsTest(shouldWaitForUServer, ts): allAEHdrs(False, False, ts, 'www.ae-1.com') allAEHdrs(False, False, ts, 'www.ae-2.com') allAEHdrs(False, False, ts, 'www.ae-3.com') + allAEHdrs(False, False, ts, 'www.ae-4.com') + allAEHdrs(False, False, ts, 'www.ae-5.com') perTsTest(True, ts) diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 991d2fda32b..90441cb54f5 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -45,6 +45,12 @@ ts.Disk.remap_config.AddLine( f"map http://www.ae-3.com http://127.0.0.1:{server.Variables.http_port}" + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-4.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-5.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') ts.Disk.records_config.update( { diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 57842341a59..4982e8cffd9 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -811,3 +811,833 @@ sessions: # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 5 proxy.config.http.normalize_ae:4 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 41 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 42 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 43 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 44 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 45 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 46 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header zstd, br, compress, gzip would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 47 ] + - [ Accept-Encoding, "zstd, br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br, compress, gzip would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 48 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 49 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 410 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 6 proxy.config.http.normalize_ae:5 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 51 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 52 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 53 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 54 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 55 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 56 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 57 ] + - [ Accept-Encoding, "zstd, compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # NOTICE: This case should load an alternate of zstd, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 58 ] + - [ Accept-Encoding, "zstd, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + # - [ X-Cache, { value: miss, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 59 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "br, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Br, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + # - [ X-Cache, { value: miss, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 510 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, br would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 511 ] + - [ Accept-Encoding, "compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 512 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header zstd;q=1.1 would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 513 ] + - [ Accept-Encoding, "zstd;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br;q=1.1 would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 514 ] + - [ Accept-Encoding, "br;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 515 ] + - [ Accept-Encoding, "zstd, br;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # NOTICE: This case should make Accept-Encoding header zstd, gzip;q=0.8 match + # the alternate of zstd, gzip Accept-Encoding header. + # However, due to the implementation of calculate_quality_of_match(), + # ATS matches the alternate of gzip Accept-Encoding header. + # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 516 ] + - [ Accept-Encoding, "zstd, gzip;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] \ No newline at end of file diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6745b65f0ae..8b5bd830f04 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -1,22 +1,22 @@ -> GET ``/obj0 HTTP/1.1 -> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -25,64 +25,78 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 -> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj2 HTTP/1.1 -> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -91,75 +105,205 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 71 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666 > Accept-Encoding: gzip;q=0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x > Accept-Encoding: gzip;q=0.666x < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 > Accept-Encoding: gzip;q=#0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 > Accept-Encoding: gzip; Q = 0.666 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 > Accept-Encoding: gzip;q=0.0 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=-0.1 > Accept-Encoding: gzip;q=-0.1 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb > Accept-Encoding: aaa, gzip;q=0.666, bbb < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb > Accept-Encoding: br ; q=0.666, bbb < HTTP/1.1 200 OK @@ -168,21 +312,21 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666 , > Accept-Encoding: aaa, gzip;q=0.666 , < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === -> POST ``/obj3 HTTP/1.1 +> POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 7`` +< Content-Length: 71 === diff --git a/tests/gold_tests/pluginTest/compress/compress.test.py b/tests/gold_tests/pluginTest/compress/compress.test.py index fd34c6f7524..4359b64a5cd 100644 --- a/tests/gold_tests/pluginTest/compress/compress.test.py +++ b/tests/gold_tests/pluginTest/compress/compress.test.py @@ -25,7 +25,8 @@ # Skip if plugins not present. # Test.SkipUnless( - Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI')) + Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI'), + Condition.HasATSFeature('TS_HAS_ZSTD')) server = Test.MakeOriginServer("server", options={'--load': f'{Test.TestDirectory}/compress_observer.py'}) @@ -43,7 +44,7 @@ "timestamp": "1469733493.993", "body": body } -for i in range(3): +for i in range(6): # add request/response to the server dictionary request_header = {"headers": f"GET /obj{i} HTTP/1.1\r\nHost: just.any.thing\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) @@ -89,6 +90,7 @@ def curl_post(ts, idx, encodingList, out_path): ts.Setup.Copy("compress.config") ts.Setup.Copy("compress2.config") +ts.Setup.Copy("compress3.config") ts.Disk.remap_config.AddLine( f'map http://ae-0/ http://127.0.0.1:{server.Variables.Port}/' + @@ -103,7 +105,16 @@ def curl_post(ts, idx, encodingList, out_path): f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') ts.Disk.remap_config.AddLine( f'map http://ae-3/ http://127.0.0.1:{server.Variables.Port}/' + - f' @plugin=compress.so @pparam={Test.RunDirectory}/compress.config') + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-4/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-5/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') out_path_counter = 0 @@ -119,12 +130,12 @@ def get_out_path(): def get_verify_command(out_path, decrompressor): - return f"{decrompressor} -c {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" + return f"{decrompressor} {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" -for i in range(3): +for i in range(6): - tr = Test.AddTestRun(f'gzip, deflate, sdch, br: {i}') + tr = Test.AddTestRun(f'gzip, deflate, sdch, br, zstd: {i}') if (waitForTs): tr.Processes.Default.StartBefore(ts) waitForTs = False @@ -133,15 +144,21 @@ def get_verify_command(out_path, decrompressor): waitForServer = False tr.Processes.Default.ReturnCode = 0 out_path = get_out_path() - tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br', out_path), ts=ts) - tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br: {i}') + tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br, zstd', out_path), ts=ts) + tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br, zstd: {i}') tr.ReturnCode = 0 if i == 0: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") elif i == 1: - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") elif i == 2: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 3: + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 4: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + elif i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") tr = Test.AddTestRun(f'gzip: {i}') tr.Processes.Default.ReturnCode = 0 @@ -149,7 +166,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, i, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip: {i}') tr.ReturnCode = 0 - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun(f'br: {i}') tr.Processes.Default.ReturnCode = 0 @@ -160,7 +177,7 @@ def get_verify_command(out_path, decrompressor): if i == 1: tr.Processes.Default.Command = f"diff {out_path} {orig_path}" else: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun(f'deflate: {i}') tr.Processes.Default.ReturnCode = 0 @@ -170,6 +187,17 @@ def get_verify_command(out_path, decrompressor): tr.ReturnCode = 0 tr.Processes.Default.Command = f"diff {out_path} {orig_path}" + tr = Test.AddTestRun(f'zstd: {i}') + tr.Processes.Default.ReturnCode = 0 + out_path = get_out_path() + tr.MakeCurlCommand(curl(ts, i, "zstd", out_path)) + tr = Test.AddTestRun(f'verify zstd: {i}') + tr.ReturnCode = 0 + if i == 4 or i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + else: + tr.Processes.Default.Command = get_verify_command(out_path, "cat") + # Test Accept-Encoding normalization. tr = Test.AddTestRun() @@ -178,7 +206,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -186,7 +214,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666x", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666x') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -194,7 +222,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=#0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=#0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -202,7 +230,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip; Q = 0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip; Q = 0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -218,7 +246,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=-0.1", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=-0.1') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -226,7 +254,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -234,7 +262,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, " br ; q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify br ; q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") +tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -242,7 +270,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666 , ", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666 , ') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # post tr = Test.AddTestRun() @@ -251,7 +279,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl_post(ts, 3, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip post') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # compress_long.log contains all the output from the curl commands. The tr removes the carriage returns for easier # readability. Curl seems to have a bug, where it will neglect to output an end of line before outputting an HTTP diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config new file mode 100644 index 00000000000..e3674f0e4cb --- /dev/null +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -0,0 +1,7 @@ +cache true +remove-accept-encoding true +compressible-content-type text/* +compressible-content-type application/x-javascript* +compressible-content-type application/javascript* +compressible-content-type application/json* +supported-algorithms zstd, br, gzip diff --git a/tests/gold_tests/pluginTest/compress/compress_userver.gold b/tests/gold_tests/pluginTest/compress/compress_userver.gold index 1ddc6a4f3b5..693bd03905e 100644 --- a/tests/gold_tests/pluginTest/compress/compress_userver.gold +++ b/tests/gold_tests/pluginTest/compress/compress_userver.gold @@ -1,15 +1,33 @@ -0/gzip, deflate, sdch, br +0/gzip, deflate, sdch, br, zstd 0/gzip 0/br 0/deflate -1/gzip, deflate, sdch, br +0/zstd +1/gzip, deflate, sdch, br, zstd 1/gzip 1/br 1/deflate -2/gzip, deflate, sdch, br +1/zstd +2/gzip, deflate, sdch, br, zstd 2/gzip 2/br 2/deflate +2/zstd +3/gzip, deflate, sdch, br, zstd +3/gzip +3/br +3/deflate +3/zstd +4/gzip, deflate, sdch, br, zstd +4/gzip +4/br +4/deflate +4/zstd +5/gzip, deflate, sdch, br, zstd +5/gzip +5/br +5/deflate +5/zstd 0/gzip;q=0.666 0/gzip;q=0.666x 0/gzip;q=#0.666 From facf384cd557737e5e1bf268c29bb4a86ef9db1d Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 01:32:52 +0100 Subject: [PATCH 02/15] Fix Accept-Encoding cache matching and update test expectations This patch fixes the Accept-Encoding quality calculation logic in the HTTP transaction cache and updates test files to reflect the corrected behavior, removing workarounds for previously broken cache matching. Cache quality calculation improvements: - Replace multiplicative quality combining with minimum quality selection for multiple content encodings - Simplify wildcard matching to return quality directly - Remove gzip-specific fallback logic that bypassed normal quality calculation and caused inconsistent cache behavior Test updates for corrected behavior: - Remove NOTICE comments describing broken cache matching behavior - Update test expectations to match correct cache responses instead of workaround responses - Fix cache hit/miss expectations for multi-encoding scenarios - Add missing Content-Encoding headers in server responses - Update response identifiers to match the actual cached alternates Specific test corrections: - "br, compress, gzip" now correctly matches "Br-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - "zstd, compress, gzip" now correctly matches "Zstd-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - "zstd, gzip;q=0.8" now correctly matches "Zstd-Gzip-Accept-Encoding" instead of falling back to "Gzip-Accept-Encoding" - Individual encoding requests now create proper cache alternates instead of incorrectly matching empty encoding alternates The previous multiplicative quality calculation (q_a * q_b) was causing unexpectedly low quality scores and incorrect cache alternate selection. The new minimum quality approach ensures that multi-encoding responses are properly cached and matched according to HTTP content negotiation standards. All test cases now pass without workarounds and demonstrate correct cache behavior for all compression algorithms (gzip, brotli, zstd) across various Accept-Encoding --- src/iocore/cache/HttpTransactCache.cc | 58 +++------ .../normalized_ae_match_vary_cache.test.py | 2 +- ...malized_ae_varied_transactions.replay.yaml | 115 ++++++++++-------- 3 files changed, 83 insertions(+), 92 deletions(-) diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 021985fff47..215b7a49ef1 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -979,60 +979,38 @@ HttpTransactCache::calculate_quality_of_accept_encoding_match(MIMEField *accept_ if (!content_field) { if (!match_accept_content_encoding("identity", accept_field, &wildcard_present, &wildcard_q, &q)) { // CE was not returned, and AE does not have identity - if (match_content_encoding(accept_field, "gzip") and match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } goto encoding_wildcard; } - // use q from identity match - } else { - // "Accept-encoding must correctly handle multiple content encoding" - // The combined quality factor is the product of all quality factors. - // (Note that there may be other possible choice, eg, min(), - // but I think multiplication is the best.) - // For example, if "content-encoding: a, b", and quality factors - // of a and b (in accept-encoding header) are q_a and q_b, resp, - // then the combined quality factor is (q_a * q_b). - // If any one of the content-encoding is not matched, - // then the q value will not be changed. - float combined_q = 1.0; + // Handle multiple content encodings - use minimum quality + float min_q = 1.0; // Start with maximum quality + bool found_match = false; + for (c_value = c_values_list.head; c_value; c_value = c_value->next) { float this_q = -1.0; if (!match_accept_content_encoding(c_value->str, accept_field, &wildcard_present, &wildcard_q, &this_q)) { goto encoding_wildcard; } - combined_q *= this_q; + if (this_q >= 0.0) { + found_match = false; + if (this_q < min_q) { + min_q = this_q; + } + } + } + if (found_match) { + q = min_q; + } else { + q = -1.0; } - q = combined_q; } encoding_wildcard: - // match the wildcard now // if ((q == -1.0) && (wildcard_present == true)) { - q = wildcard_q; - } - ///////////////////////////////////////////////////////////////////////// - // there was an Accept-Encoding, but it didn't match anything, at // - // any quality level --- if this is an identity-coded document, that's // - // still okay, but otherwise, this is just not a match at all. // - ///////////////////////////////////////////////////////////////////////// - if ((q == -1.0) && is_identity_encoding) { - if (match_content_encoding(accept_field, "gzip")) { - if (match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } else { - // always try to fetch GZIP content if we have not tried sending AE before - return -1.0f; - } - } else if (cached_accept_field && !match_content_encoding(cached_accept_field, "gzip")) { - return 0.001f; - } else { - return -1.0f; - } + return wildcard_q; } - // q = (float)-1.0; - return (q); + + return q; } /** diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 90441cb54f5..f83ae463fb9 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -59,7 +59,7 @@ 'proxy.config.http.response_via_str': 3, # the following variables could affect the results of alternate cache matching, # define them with their default values explicitly - 'proxy.config.cache.limits.http.max_alts': 5, + 'proxy.config.cache.limits.http.max_alts': 6, 'proxy.config.http.cache.ignore_accept_mismatch': 2, 'proxy.config.http.cache.ignore_accept_language_mismatch': 2, 'proxy.config.http.cache.ignore_accept_encoding_mismatch': 2, diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 4982e8cffd9..4910af81027 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -96,7 +96,7 @@ sessions: - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header deflate would match the alternate of empty Accept-Encoding header + # load an alternate of deflate Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -110,16 +110,25 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, deflate ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Deflate-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Deflate-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress would match the alternate of empty Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -133,14 +142,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # load an alternate of gzip Accept-Encoding header - client-request: @@ -162,6 +180,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -176,7 +195,7 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress, gzip would match the alternate of gzip Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -190,14 +209,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Case 2 proxy.config.http.normalize_ae:1 @@ -322,6 +350,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -458,6 +487,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -492,6 +522,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -651,6 +682,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -686,6 +718,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, gzip ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: @@ -699,10 +732,6 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -722,6 +751,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, "br, gzip" ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] @@ -733,10 +763,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header - client-request: @@ -784,11 +812,6 @@ sessions: - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header br, gzip;q=0.8 match - # the alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -808,8 +831,7 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] # Case 5 proxy.config.http.normalize_ae:4 @@ -915,6 +937,7 @@ sessions: headers: fields: - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, zstd ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] @@ -954,6 +977,7 @@ sessions: headers: fields: - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, br ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] @@ -994,6 +1018,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -1224,6 +1249,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, zstd ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Zstd-Accept-Encoding ] @@ -1264,6 +1290,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, br ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] content: @@ -1304,6 +1331,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, gzip ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: encoding: plain @@ -1341,6 +1369,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, zstd ] - [ Connection, close ] - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] content: @@ -1354,10 +1383,6 @@ sessions: - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of zstd, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1384,6 +1409,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, zstd ] - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] content: encoding: plain @@ -1393,15 +1419,9 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1428,6 +1448,7 @@ sessions: - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] - [ Connection, close ] + - [ Content-Encoding, br ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] content: encoding: plain @@ -1437,10 +1458,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header - client-request: @@ -1609,11 +1628,6 @@ sessions: - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header zstd, gzip;q=0.8 match - # the alternate of zstd, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 5 - client-request: method: "GET" version: "1.1" @@ -1638,6 +1652,5 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] \ No newline at end of file + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] From eda7c29e722c13248e8d56865d101c8be5f73067 Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 13:42:50 +0100 Subject: [PATCH 03/15] Fix case sensitivity in ZSTD package find command --- CMakeLists.txt | 4 +-- cmake/Findzstd.cmake | 28 ++++++++--------- .../pluginTest/compress/compress.gold | 30 +++++++++---------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index df0f61910f9..0defeeb05e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,8 +341,8 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(ZSTD) -if(ZSTD_FOUND) +find_package(zstd) +if(zstd_FOUND) set(HAVE_ZSTD_H TRUE) endif() diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake index d2a86d0132e..17587dca39a 100644 --- a/cmake/Findzstd.cmake +++ b/cmake/Findzstd.cmake @@ -20,34 +20,34 @@ # # This will define the following variables # -# ZSTD_FOUND -# ZSTD_LIBRARY -# ZSTD_INCLUDE_DIRS +# zstd_FOUND +# zstd_LIBRARY +# zstd_INCLUDE_DIRS # # and the following imported target # # zstd::zstd # -find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) +find_path(zstd_INCLUDE_DIR NAMES zstd.h) -find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) -find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) +find_library(zstd_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(zstd_LIBRARY_RELEASE NAMES zstd zstd_static) -mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +mark_as_advanced(zstd_LIBRARY zstd_INCLUDE_DIR) include(SelectLibraryConfigurations) -select_library_configurations(ZSTD) +select_library_configurations(zstd) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARY zstd_INCLUDE_DIR) -if(ZSTD_FOUND) - set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") +if(zstd_FOUND) + set(zstd_INCLUDE_DIRS "${zstd_INCLUDE_DIR}") endif() -if(ZSTD_FOUND AND NOT TARGET zstd::zstd) +if(zstd_FOUND AND NOT TARGET zstd::zstd) add_library(zstd::zstd INTERFACE IMPORTED) - target_include_directories(zstd::zstd INTERFACE ${ZSTD_INCLUDE_DIRS}) - target_link_libraries(zstd::zstd INTERFACE "${ZSTD_LIBRARY}") + target_include_directories(zstd::zstd INTERFACE ${zstd_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${zstd_LIBRARY}") endif() diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 8b5bd830f04..1a7014bc9bc 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -175,7 +175,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -216,7 +216,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -249,7 +249,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -258,7 +258,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -267,7 +267,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -276,7 +276,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -292,7 +292,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -301,7 +301,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -319,7 +319,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -328,5 +328,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === From 21e27bffa5e9522f6314185b8b80a8d467c46c9f Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 22:20:01 +0100 Subject: [PATCH 04/15] Add ZSTD version output to compile-time features --- src/traffic_layout/CMakeLists.txt | 4 ++++ src/traffic_layout/info.cc | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index 56766955c62..b86d4f2573a 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -31,6 +31,10 @@ if(HAVE_BROTLI_ENCODE_H) target_link_libraries(traffic_layout PRIVATE brotli::brotlienc) endif() +if(HAVE_ZSTD_H) + target_link_libraries(traffic_layout PRIVATE zstd::zstd) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 541985112cc..6e5abd71046 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -49,6 +49,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built static void print_feature(std::string_view name, int value, bool json, bool last = false) @@ -208,6 +212,11 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif +#if HAVE_ZSTD_H + print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); +#else + print_var("zstd", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); From 2a2acf66d667251deb5520e73c1740bcc0d30c6b Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 15 Jul 2025 23:52:16 +0100 Subject: [PATCH 05/15] Add ZSTD support to Accept-Encoding and Content-Encoding headers --- src/proxy/http3/QPACK.cc | 3 +- .../pluginTest/compress/compress.gold | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/proxy/http3/QPACK.cc b/src/proxy/http3/QPACK.cc index bcfa4c70b8d..dfdd2d278b3 100644 --- a/src/proxy/http3/QPACK.cc +++ b/src/proxy/http3/QPACK.cc @@ -69,7 +69,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {":status", "503" }, {"accept", "*/*" }, {"accept", "application/dns-message" }, - {"accept-encoding", "gzip, deflate, br" }, + {"accept-encoding", "gzip, deflate, br, zstd" }, {"accept-ranges", "bytes" }, {"access-control-allow-headers", "cache-control" }, {"access-control-allow-headers", "content-type" }, @@ -80,6 +80,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {"cache-control", "no-cache" }, {"cache-control", "no-store" }, {"cache-control", "public, max-age=31536000" }, + {"content-encoding", "zstd" }, {"content-encoding", "br" }, {"content-encoding", "gzip" }, {"content-type", "application/dns-message" }, diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 1a7014bc9bc..6501a0e7318 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -165,6 +165,7 @@ > Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -175,7 +176,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -198,6 +199,7 @@ > Accept-Encoding: zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -206,6 +208,7 @@ > Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -216,7 +219,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -239,6 +242,7 @@ > Accept-Encoding: zstd < HTTP/1.1 200 OK < Content-Type: text/javascript +< Content-Encoding: zstd < Vary: Accept-Encoding < Content-Length: 64 === @@ -249,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -258,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -267,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -276,7 +280,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -292,7 +296,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -301,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -319,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -328,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 72 +< Content-Length: 71 === From 536f920731b6c8e7b5a29a5b4843a9218f98b4cf Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 00:09:55 +0100 Subject: [PATCH 06/15] Add ZSTD length definition to HTTP value initialization --- src/api/InkAPIInternal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index 9a4bb620ab1..8bb20a2b38a 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -774,6 +774,7 @@ api_init() TS_HTTP_LEN_DEFLATE = static_cast(HTTP_VALUE_DEFLATE.length()); TS_HTTP_LEN_GZIP = static_cast(HTTP_VALUE_GZIP.length()); TS_HTTP_LEN_BROTLI = static_cast(HTTP_VALUE_BROTLI.length()); + TS_HTTP_LEN_ZSTD = static_cast(HTTP_VALUE_ZSTD.length()); TS_HTTP_LEN_IDENTITY = static_cast(HTTP_VALUE_IDENTITY.length()); TS_HTTP_LEN_KEEP_ALIVE = static_cast(HTTP_VALUE_KEEP_ALIVE.length()); TS_HTTP_LEN_MAX_AGE = static_cast(HTTP_VALUE_MAX_AGE.length()); From 9a5432d04b969463f321207bbd8f096ad5e9c757 Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 00:57:41 +0100 Subject: [PATCH 07/15] Update ZSTD compression level from 6 to 10 for improved performance --- plugins/compress/compress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index b181d675722..04a4620989e 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -75,7 +75,7 @@ const int BROTLI_LGW = 16; #endif #if HAVE_ZSTD_H -const int ZSTD_COMPRESSION_LEVEL = 6; +const int ZSTD_COMPRESSION_LEVEL = 12; #endif static const char *global_hidden_header_name = nullptr; From ab79771971fadb4365efc8910490b746b0b09e1b Mon Sep 17 00:00:00 2001 From: jake champion Date: Wed, 16 Jul 2025 09:39:08 +0100 Subject: [PATCH 08/15] compress: Add configurable compression levels - Add support for configurable compression levels per host: * gzip-compression-level (1-9, default 6) * brotli-compression-level (0-11, default 6) * brotli-lgwin (10-24, default 16) * zstd-compression-level (1-22, default 12) This enables fine-tuning compression performance vs. speed trade-offs on a per-host basis --- doc/admin-guide/plugins/compress.en.rst | 50 ++++++++++++++++- plugins/compress/README | 2 +- plugins/compress/compress.cc | 29 +++------- plugins/compress/configuration.cc | 54 +++++++++++++++++- plugins/compress/configuration.h | 55 ++++++++++++++++++- plugins/compress/sample.compress.config | 28 ++++++++++ .../pluginTest/compress/compress.gold | 4 +- .../pluginTest/compress/compress2.config | 3 + .../pluginTest/compress/compress3.config | 4 ++ 9 files changed, 202 insertions(+), 27 deletions(-) diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 686445008fc..338e6101c11 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -211,6 +211,41 @@ be considered, if it is ``2``, only br or gzip will be considered, if it is ``4` only zstd, br, or gzip will be considered, and if it is ``5``, all combinations of zstd, br, and gzip will be considered. +gzip-compression-level +----------------------- + +Sets the compression level for gzip compression. Valid values are 1-9, where +1 is fastest compression (lowest compression ratio) and 9 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-compression-level +------------------------- + +Sets the compression level for Brotli compression. Valid values are 0-11, where +0 is fastest compression (lowest compression ratio) and 11 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-lgwin +------------ + +Sets the window size for Brotli compression. Valid values are 10-24, where +larger values provide better compression but use more memory. The default is 16. +This parameter controls the sliding window size used during compression: + +- 10: 1KB window (fastest, least memory) +- 16: 64KB window (default, good balance) +- 24: 16MB window (slowest, most memory, best compression) + +zstd-compression-level +---------------------- + +Sets the compression level for Zstandard compression. Valid values are 1-22, where +1 is fastest compression (lowest compression ratio) and 22 is slowest compression +(highest compression ratio). The default is 12, which provides an excellent +balance between compression speed and ratio for web content. + Examples ======== @@ -226,6 +261,10 @@ might create a configuration with the following options:: compressible-status-code 200, 206 minimum-content-length 860 flush false + gzip-compression-level 6 + brotli-compression-level 6 + brotli-lgwin 16 + zstd-compression-level 12 # Now set a configuration for www.example.com [www.example.com] @@ -243,13 +282,15 @@ might create a configuration with the following options:: flush true supported-algorithms gzip,deflate - # Supports brotli compression + # Supports brotli compression with custom settings [brotli.compress.com] enabled true compressible-content-type text/* compressible-content-type application/json flush true supported-algorithms br,gzip + brotli-compression-level 8 + brotli-lgwin 20 # Supports zstd compression for high efficiency [zstd.compress.com] @@ -259,14 +300,19 @@ might create a configuration with the following options:: compressible-content-type application/javascript flush true supported-algorithms zstd,gzip + zstd-compression-level 15 - # Supports all compression algorithms + # Supports all compression algorithms with optimized settings [all.compress.com] enabled true compressible-content-type text/* compressible-content-type application/json flush true supported-algorithms zstd,br,gzip,deflate + gzip-compression-level 7 + brotli-compression-level 9 + brotli-lgwin 18 + zstd-compression-level 10 # This origin does it all [bar.example.com] diff --git a/plugins/compress/README b/plugins/compress/README index e89070f9e54..f963a8e05e9 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -24,4 +24,4 @@ compress.so /sample.compress.config After modifying plugin.config, restart traffic server (sudo traffic_ctl server restart) the configuration is re-read when a management update is given (sudo traffic_ctl config reload) -See sample.config.compress for an example configuration and the options that are available +See sample.compress.config for an example configuration and the options that are available diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 04a4620989e..ffb3d793d76 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -65,18 +65,7 @@ namespace compress_ns DbgCtl dbg_ctl{TAG}; } -const int ZLIB_COMPRESSION_LEVEL = 6; -const char *dictionary = nullptr; - -// brotli compression quality 1-11. Testing proved level '6' -#if HAVE_BROTLI_ENCODE_H -const int BROTLI_COMPRESSION_LEVEL = 6; -const int BROTLI_LGW = 16; -#endif - -#if HAVE_ZSTD_H -const int ZSTD_COMPRESSION_LEVEL = 12; -#endif +const char *dictionary = nullptr; static const char *global_hidden_header_name = nullptr; @@ -151,7 +140,7 @@ static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t u #endif static Data * -data_alloc(int compression_type, int compression_algorithms) +data_alloc(int compression_type, int compression_algorithms, HostConfiguration *hc) { Data *data; int err; @@ -164,6 +153,7 @@ data_alloc(int compression_type, int compression_algorithms) data->state = transform_state_initialized; data->compression_type = compression_type; data->compression_algorithms = compression_algorithms; + data->hc = hc; data->zstrm.next_in = Z_NULL; data->zstrm.avail_in = 0; data->zstrm.total_in = 0; @@ -180,7 +170,7 @@ data_alloc(int compression_type, int compression_algorithms) window_bits = WINDOW_BITS_DEFLATE; } - err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); + err = deflateInit2(&data->zstrm, data->hc->zlib_compression_level(), Z_DEFLATED, window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); if (err != Z_OK) { fatal("gzip-transform: ERROR: deflateInit (%d)!", err); @@ -200,8 +190,8 @@ data_alloc(int compression_type, int compression_algorithms) if (!data->bstrm.br) { fatal("Brotli Encoder Instance Failed"); } - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, BROTLI_COMPRESSION_LEVEL); - BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, BROTLI_LGW); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_QUALITY, data->hc->brotli_compression_level()); + BrotliEncoderSetParameter(data->bstrm.br, BROTLI_PARAM_LGWIN, data->hc->brotli_lgw_size()); data->bstrm.next_in = nullptr; data->bstrm.avail_in = 0; data->bstrm.total_in = 0; @@ -523,7 +513,7 @@ zstd_compress_init(Data *data) } // Set compression level - size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL); + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); if (ZSTD_isError(result)) { error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); return; @@ -536,7 +526,7 @@ zstd_compress_init(Data *data) return; } - debug("zstd compression context initialized with level %d", ZSTD_COMPRESSION_LEVEL); + debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); } static void @@ -1058,9 +1048,8 @@ compress_transform_add(TSHttpTxn txnp, HostConfiguration *hc, int compress_type, } connp = TSTransformCreate(compress_transform, txnp); - data = data_alloc(compress_type, algorithms); + data = data_alloc(compress_type, algorithms, hc); data->txn = txnp; - data->hc = hc; TSContDataSet(connp, data); TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 41b35964eac..d4311ec286c 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -108,7 +108,11 @@ enum ParserState { kParseRangeRequest, kParseFlush, kParseAllow, - kParseMinimumContentLength + kParseMinimumContentLength, + kParseGzipCompressionLevel, + kPaseBrotliCompressionLevel, + kParseBrotliLGWSize, + kParseZstdCompressionLevel, }; void @@ -389,6 +393,14 @@ Configuration::Parse(const char *path) state = kParseStart; } else if (token == "minimum-content-length") { state = kParseMinimumContentLength; + } else if (token == "gzip-compression-level") { + state = kParseGzipCompressionLevel; + } else if (token == "brotli-compression-level") { + state = kPaseBrotliCompressionLevel; + } else if (token == "brotli-lgwin") { + state = kParseBrotliLGWSize; + } else if (token == "zstd-compression-level") { + state = kParseZstdCompressionLevel; } else { warning("failed to interpret \"%s\" at line %zu", token.c_str(), lineno); } @@ -425,6 +437,46 @@ Configuration::Parse(const char *path) current_host_configuration->set_minimum_content_length(strtoul(token.c_str(), nullptr, 10)); state = kParseStart; break; + case kParseGzipCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 9) { + error("gzip-compression-level must be between 1 and 9, got %d", level); + } else { + current_host_configuration->set_gzip_compression_level(level); + } + state = kParseStart; + break; + } + case kPaseBrotliCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 0 || level > 11) { + error("brotli-compression-level must be between 0 and 11, got %d", level); + } else { + current_host_configuration->set_brotli_compression_level(level); + } + state = kParseStart; + break; + } + case kParseBrotliLGWSize: { + int lgw = strtol(token.c_str(), nullptr, 10); + if (lgw < 10 || lgw > 24) { + error("brotli-lgwin must be between 10 and 24, got %d", lgw); + } else { + current_host_configuration->set_brotli_lgw_size(lgw); + } + state = kParseStart; + break; + } + case kParseZstdCompressionLevel: { + int level = strtol(token.c_str(), nullptr, 10); + if (level < 1 || level > 22) { + error("zstd-compression-level must be between 1 and 22, got %d", level); + } else { + current_host_configuration->set_zstd_compression_level(level); + } + state = kParseStart; + break; + } } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 2140d5a3580..0db81784199 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -59,7 +59,11 @@ class HostConfiguration : private atscppapi::noncopyable remove_accept_encoding_(false), flush_(false), compression_algorithms_(ALGORITHM_GZIP), - minimum_content_length_(1024) + minimum_content_length_(1024), + zlib_compression_level_(6), + brotli_compression_level_(6), + brotli_lgw_size_(16), + zstd_compression_level_(12) { } @@ -130,6 +134,51 @@ class HostConfiguration : private atscppapi::noncopyable minimum_content_length_ = x; } + unsigned int + zlib_compression_level() const + { + return zlib_compression_level_; + } + + void + set_gzip_compression_level(int level) + { + zlib_compression_level_ = level; + } + + unsigned int + brotli_compression_level() const + { + return brotli_compression_level_; + } + void + set_brotli_compression_level(int level) + { + brotli_compression_level_ = level; + } + + unsigned int + brotli_lgw_size() const + { + return brotli_lgw_size_; + } + void + set_brotli_lgw_size(unsigned int lgw) + { + brotli_lgw_size_ = lgw; + } + + int + zstd_compression_level() const + { + return zstd_compression_level_; + } + void + set_zstd_compression_level(int level) + { + zstd_compression_level_ = level; + } + void update_defaults(); void add_allow(const std::string &allow); void add_compressible_content_type(const std::string &content_type); @@ -149,6 +198,10 @@ class HostConfiguration : private atscppapi::noncopyable bool flush_; int compression_algorithms_; unsigned int minimum_content_length_; + unsigned int zlib_compression_level_; + unsigned int brotli_compression_level_; + unsigned int brotli_lgw_size_; + int zstd_compression_level_; RangeRequestCtrl range_request_ctl_ = RangeRequestCtrl::NO_COMPRESSION; StringContainer compressible_content_types_; diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index 8b0eaaf5be8..451f8958349 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -37,6 +37,22 @@ # minimum-content-length: minimum content length for compression to be enabled (in bytes) # - this setting only applies if the origin response has a Content-Length header # +# gzip-compression-level: compression level for gzip (1-9, default 6) +# - 1 is fastest compression (lowest compression ratio) +# - 9 is slowest compression (highest compression ratio) +# +# brotli-compression-level: compression level for brotli (0-11, default 6) +# - 0 is fastest compression (lowest compression ratio) +# - 11 is slowest compression (highest compression ratio) +# +# brotli-lgwin: window size for brotli compression (10-24, default 16) +# - larger values provide better compression but use more memory +# - 10: 1KB window, 16: 64KB window, 24: 16MB window +# +# zstd-compression-level: compression level for zstandard (1-22, default 12) +# - 1 is fastest compression (lowest compression ratio) +# - 22 is slowest compression (highest compression ratio) +# ###################################################################### #first, we configure the default/global plugin behaviour @@ -58,6 +74,12 @@ minimum-content-length 1024 #supported algorithms supported-algorithms br,gzip,zstd +# Compression level settings (optional) +gzip-compression-level 6 +brotli-compression-level 6 +brotli-lgwin 16 +zstd-compression-level 12 + #override the global configuration for a host. #www.foo.nl does NOT inherit anything [www.foo.nl] @@ -68,6 +90,12 @@ compressible-content-type text/* compressible-status-code 200,206,409 minimum-content-length 1024 +# Custom compression settings for this host +gzip-compression-level 8 +brotli-compression-level 9 +brotli-lgwin 20 +zstd-compression-level 15 + allow /this/*.js allow !/notthis/*.js allow !/notthat* diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6501a0e7318..575c896e56e 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -185,7 +185,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 46 +< Content-Length: 47 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/deflate @@ -228,7 +228,7 @@ < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding -< Content-Length: 46 +< Content-Length: 47 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/deflate diff --git a/tests/gold_tests/pluginTest/compress/compress2.config b/tests/gold_tests/pluginTest/compress/compress2.config index c808d87fcfd..8fa674c687b 100644 --- a/tests/gold_tests/pluginTest/compress/compress2.config +++ b/tests/gold_tests/pluginTest/compress/compress2.config @@ -5,3 +5,6 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms gzip, br +gzip-compression-level 7 +brotli-compression-level 8 +brotli-lgwin 18 diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config index e3674f0e4cb..54ff0eed2d8 100644 --- a/tests/gold_tests/pluginTest/compress/compress3.config +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -5,3 +5,7 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms zstd, br, gzip +gzip-compression-level 5 +brotli-compression-level 7 +brotli-lgwin 17 +zstd-compression-level 10 From d94321de61b80c323f87693928e6e3223ee288f4 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Wed, 16 Jul 2025 13:15:00 +0100 Subject: [PATCH 09/15] Update compress.gold --- .../pluginTest/compress/compress.gold | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 575c896e56e..970ba0084d1 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -14,7 +14,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br @@ -46,7 +46,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip @@ -55,7 +55,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br @@ -94,7 +94,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br @@ -135,7 +135,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/br @@ -176,7 +176,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-4/obj4 HTTP/1.1 > X-Ats-Compress-Test: 4/br @@ -219,7 +219,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-5/obj5 HTTP/1.1 > X-Ats-Compress-Test: 5/br @@ -253,7 +253,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x @@ -262,7 +262,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 @@ -271,7 +271,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 @@ -280,7 +280,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 @@ -296,7 +296,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb @@ -305,7 +305,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb @@ -323,7 +323,7 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === > POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip @@ -332,5 +332,5 @@ < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding -< Content-Length: 71 +< Content-Length: 72 === From 729fbac68a744036bc38c09ff44f11a29bd47dde Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Tue, 22 Jul 2025 12:22:22 +0100 Subject: [PATCH 10/15] Update compress.gold From 17e3c6b49373f5b35977f420719f18d0ec3b3efb Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:31:17 +0100 Subject: [PATCH 11/15] Use upstream zstd CMake; remove Findzstd.cmake Switch to the upstream zstd CMake package and drop our custom Find-module. - Use `find_package(zstd CONFIG QUIET)` and set `HAVE_ZSTD_H` when found - Provide a compatibility target `zstd::zstd` that aliases `zstd::libzstd_shared`/`zstd::libzstd_static`/`zstd::libzstd` to keep existing link lines working - Remove `cmake/Findzstd.cmake` and rely on distro-provided configs This reduces maintenance and prefers the canonical package configuration while retaining compatibility with current linkage in the tree. --- CMakeLists.txt | 20 ++++++++++++++++- cmake/Findzstd.cmake | 53 -------------------------------------------- 2 files changed, 19 insertions(+), 54 deletions(-) delete mode 100644 cmake/Findzstd.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0defeeb05e7..bbcc2bc2745 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,9 +341,27 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(zstd) +find_package(zstd CONFIG QUIET) if(zstd_FOUND) set(HAVE_ZSTD_H TRUE) + + # Provide a compatibility target name if the upstream package does not export it + # Our code links against `zstd::zstd`; upstream zstd usually exports + # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. + if(NOT TARGET zstd::zstd) + if(TARGET zstd::libzstd_shared) + set(_zstd_target zstd::libzstd_shared) + elseif(TARGET zstd::libzstd_static) + set(_zstd_target zstd::libzstd_static) + elseif(TARGET zstd::libzstd) + set(_zstd_target zstd::libzstd) + endif() + if(DEFINED _zstd_target) + add_library(zstd_zstd INTERFACE) + target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) + add_library(zstd::zstd ALIAS zstd_zstd) + endif() + endif() endif() # ncurses is used in traffic_top diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake deleted file mode 100644 index 17587dca39a..00000000000 --- a/cmake/Findzstd.cmake +++ /dev/null @@ -1,53 +0,0 @@ -####################### -# -# Licensed to the Apache Software Foundation (ASF) under one or more contributor license -# agreements. See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# -####################### - -# Findzstd.cmake -# -# This will define the following variables -# -# zstd_FOUND -# zstd_LIBRARY -# zstd_INCLUDE_DIRS -# -# and the following imported target -# -# zstd::zstd -# - -find_path(zstd_INCLUDE_DIR NAMES zstd.h) - -find_library(zstd_LIBRARY_DEBUG NAMES zstdd zstd_staticd) -find_library(zstd_LIBRARY_RELEASE NAMES zstd zstd_static) - -mark_as_advanced(zstd_LIBRARY zstd_INCLUDE_DIR) - -include(SelectLibraryConfigurations) -select_library_configurations(zstd) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARY zstd_INCLUDE_DIR) - -if(zstd_FOUND) - set(zstd_INCLUDE_DIRS "${zstd_INCLUDE_DIR}") -endif() - -if(zstd_FOUND AND NOT TARGET zstd::zstd) - add_library(zstd::zstd INTERFACE IMPORTED) - target_include_directories(zstd::zstd INTERFACE ${zstd_INCLUDE_DIRS}) - target_link_libraries(zstd::zstd INTERFACE "${zstd_LIBRARY}") -endif() From 91029c557572da3b6c45cc6170924609ed3ee439 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:38:44 +0100 Subject: [PATCH 12/15] Ensure HAVE_ZSTD_H is always set to a value --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bbcc2bc2745..8bae90bea49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,6 @@ find_package(ZLIB REQUIRED) find_package(zstd CONFIG QUIET) if(zstd_FOUND) - set(HAVE_ZSTD_H TRUE) # Provide a compatibility target name if the upstream package does not export it # Our code links against `zstd::zstd`; upstream zstd usually exports @@ -360,8 +359,13 @@ if(zstd_FOUND) add_library(zstd_zstd INTERFACE) target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) add_library(zstd::zstd ALIAS zstd_zstd) + set(HAVE_ZSTD_H TRUE) + else() + set(HAVE_ZSTD_H FALSE) endif() endif() +else() + set(HAVE_ZSTD_H FALSE) endif() # ncurses is used in traffic_top From c7389c59e7afbc4aa662f301741d7d660ac07a61 Mon Sep 17 00:00:00 2001 From: jake champion Date: Mon, 18 Aug 2025 09:58:50 +0100 Subject: [PATCH 13/15] place HAVE_ZSTD_H nearer the other cmakedefine calls --- include/tscore/ink_config.h.cmake.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 2d515338c8a..211174465f1 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -46,6 +46,7 @@ #cmakedefine HAVE_POSIX_FADVISE 1 #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 +#cmakedefine HAVE_ZSTD_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 @@ -187,5 +188,3 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@; #cmakedefine YAMLCPP_LIB_VERSION "@YAMLCPP_LIB_VERSION@" #cmakedefine01 TS_HAS_CRIPTS - -#cmakedefine HAVE_ZSTD_H 1 From 4608badde2130690bd72e4c08a7b0f5abf4bafe7 Mon Sep 17 00:00:00 2001 From: Jake Champion Date: Tue, 19 Aug 2025 08:33:35 +0100 Subject: [PATCH 14/15] Apply suggestions from code review Co-authored-by: JosiahWI <41302989+JosiahWI@users.noreply.github.com> --- src/traffic_layout/info.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 6e5abd71046..0a4f44491c0 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -95,7 +95,7 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif -#if HAVE_ZSTD_H +#ifdef HAVE_ZSTD_H print_feature("TS_HAS_ZSTD", 1, json); #else print_feature("TS_HAS_ZSTD", 0, json); @@ -212,7 +212,7 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif -#if HAVE_ZSTD_H +#ifdef HAVE_ZSTD_H print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); #else print_var("zstd", undef, json); From 03e647a7ebb7c5dbcb6552fe11cd34a527a75e4f Mon Sep 17 00:00:00 2001 From: jake champion Date: Tue, 19 Aug 2025 13:35:49 +0100 Subject: [PATCH 15/15] Renames ZSTD compression functions for consistency Updates function names from zstd_compress_* to zstd_transform_* to align with naming conventions used by other compression algorithms in the codebase. Improves code consistency and maintainability by standardizing function naming patterns across different compression implementations. --- plugins/compress/compress.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index ffb3d793d76..cfece1660e9 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -134,9 +134,9 @@ handle_range_request(TSMBuffer req_buf, TSMLoc req_loc, HostConfiguration *hc) // Forward declarations for ZSTD compression functions #if HAVE_ZSTD_H -static void zstd_compress_init(Data *data); -static void zstd_compress_finish(Data *data); -static void zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length); +static void zstd_transform_init(Data *data); +static void zstd_transform_finish(Data *data); +static void zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); #endif static Data * @@ -390,7 +390,7 @@ compress_transform_init(TSCont contp, Data *data) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD) { - zstd_compress_init(data); + zstd_transform_init(data); if (!data->zstrm_zstd.cctx) { TSError("Failed to create Zstandard compression context"); return; @@ -505,7 +505,7 @@ brotli_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_l #if HAVE_ZSTD_H static void -zstd_compress_init(Data *data) +zstd_transform_init(Data *data) { if (!data->zstrm_zstd.cctx) { error("Failed to initialize Zstd compression context"); @@ -530,7 +530,7 @@ zstd_compress_init(Data *data) } static void -zstd_compress_finish(Data *data) +zstd_transform_finish(Data *data) { if (data->state == transform_state_output) { TSIOBufferBlock downstream_blkp; @@ -570,7 +570,7 @@ zstd_compress_finish(Data *data) } static void -zstd_compress_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +zstd_transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) { TSIOBufferBlock downstream_blkp; int64_t downstream_length; @@ -633,7 +633,7 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { - zstd_compress_one(data, upstream_buffer, upstream_length); + zstd_transform_one(data, upstream_buffer, upstream_length); } else #endif #if HAVE_BROTLI_ENCODE_H @@ -725,7 +725,7 @@ compress_transform_finish(Data *data) { #if HAVE_ZSTD_H if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { - zstd_compress_finish(data); + zstd_transform_finish(data); debug("compress_transform_finish: zstd compression finish"); } else #endif