diff --git a/auto/compression b/auto/compression new file mode 100644 index 000000000..684242f25 --- /dev/null +++ b/auto/compression @@ -0,0 +1,96 @@ + +# Copyright (C) Alejandro Colomar +# Copyright (C) Andrew Clayton +# Copyright (C) NGINX, Inc. + + +NXT_HAVE_ZLIB=no +NXT_ZLIB_CFLAGS= +NXT_ZLIB_LIBS= + +NXT_HAVE_ZSTD=no +NXT_ZSTD_CFLAGS= +NXT_ZSTD_LIBS= + +NXT_HAVE_BROTLI=no +NXT_BROTLI_CFLAGS= +NXT_BROTLI_LIBS= + + +if [ $NXT_ZLIB = YES ]; then + NXT_ZLIB_CFLAGS="$(pkgconf --cflags-only-I zlib 2>/dev/null || echo "")" + NXT_ZLIB_LIBS="$(pkgconf --libs zlib 2>/dev/null || echo "-lz")" + + nxt_feature="zlib" + nxt_feature_name=NXT_HAVE_ZLIB + nxt_feature_run=no + nxt_feature_incs=$NXT_ZLIB_CFLAGS + nxt_feature_libs=$NXT_ZLIB_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + puts(zlibVersion()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_ZLIB=YES + echo " + zlib version: $(pkgconf --modversion zlib)" + fi +fi + + +if [ $NXT_ZSTD = YES ]; then + NXT_ZSTD_CFLAGS="$(pkgconf --cflags-only-I libzstd 2>/dev/null || echo "")" + NXT_ZSTD_LIBS="$(pkgconf --libs libzstd 2>/dev/null || echo "-lzstd")" + + nxt_feature="zstd" + nxt_feature_name=NXT_HAVE_ZSTD + nxt_feature_run=no + nxt_feature_incs=$NXT_ZSTD_CFLAGS + nxt_feature_libs=$NXT_ZSTD_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + printf(\"zstd version: %u\n\", ZSTD_versionNumber()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_ZSTD=YES + echo " + zstd version: $(pkgconf --modversion libzstd)" + fi +fi + + +if [ $NXT_BROTLI = YES ]; then + NXT_BROTLI_CFLAGS="$(pkgconf --cflags-only-I libbrotlienc 2>/dev/null || echo "")" + NXT_BROTLI_LIBS="$(pkgconf --libs libbrotlienc 2>/dev/null || echo "-lbrotlienc")" + + nxt_feature="brotli" + nxt_feature_name=NXT_HAVE_BROTLI + nxt_feature_run=no + nxt_feature_incs=$NXT_BROTLI_CFLAGS + nxt_feature_libs=$NXT_BROTLI_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + printf(\"brotli version: %d\n\", + BrotliEncoderVersion()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_BROTLI=YES + echo " + brotli version: $(pkgconf --modversion libbrotlienc)" + fi +fi diff --git a/auto/help b/auto/help index 948547623..1093256d8 100644 --- a/auto/help +++ b/auto/help @@ -50,6 +50,10 @@ cat << END --openssl enable OpenSSL library usage + --zlib enable zlib compression + --zstd enable zstd compression + --brotli enable brotli compression + --njs enable njs library usage --otel enable otel library usage diff --git a/auto/options b/auto/options index 7aa7a73a9..f19948588 100644 --- a/auto/options +++ b/auto/options @@ -26,6 +26,10 @@ NXT_GNUTLS=NO NXT_CYASSL=NO NXT_POLARSSL=NO +NXT_ZLIB=NO +NXT_ZSTD=NO +NXT_BROTLI=NO + NXT_NJS=NO NXT_OTEL=NO @@ -112,6 +116,10 @@ do --cyassl) NXT_CYASSL=YES ;; --polarssl) NXT_POLARSSL=YES ;; + --zlib) NXT_ZLIB=YES ;; + --zstd) NXT_ZSTD=YES ;; + --brotli) NXT_BROTLI=YES ;; + --njs) NXT_NJS=YES ;; --otel) NXT_OTEL=YES ;; diff --git a/auto/sources b/auto/sources index 027403970..28cdc8344 100644 --- a/auto/sources +++ b/auto/sources @@ -107,6 +107,7 @@ NXT_LIB_SRCS=" \ src/nxt_http_websocket.c \ src/nxt_h1proto_websocket.c \ src/nxt_fs.c \ + src/nxt_http_compression.c \ " @@ -215,6 +216,21 @@ if [ $NXT_POLARSSL = YES ]; then fi +if [ "$NXT_HAVE_ZLIB" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zlib.c" +fi + + +if [ "$NXT_HAVE_ZSTD" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zstd.c" +fi + + +if [ "$NXT_HAVE_BROTLI" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_brotli.c" +fi + + if [ "$NXT_REGEX" = "YES" ]; then if [ "$NXT_HAVE_PCRE2" = "YES" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE2_SRCS" diff --git a/auto/summary b/auto/summary index eba88be4c..4a0888a62 100644 --- a/auto/summary +++ b/auto/summary @@ -28,6 +28,9 @@ Unit configuration summary: IPv6 support: .............. $NXT_INET6 Unix domain sockets support: $NXT_UNIX_DOMAIN TLS support: ............... $NXT_OPENSSL + zlib support: .............. $NXT_ZLIB + zstd support: .............. $NXT_ZSTD + brotli support: ............ $NXT_BROTLI Regex support: ............. $NXT_REGEX njs support: ............... $NXT_NJS otel support: .............. $NXT_OTEL diff --git a/configure b/configure index 05a992ee2..aba56e8ae 100755 --- a/configure +++ b/configure @@ -127,6 +127,7 @@ NXT_LIBRT= . auto/unix . auto/os/conf . auto/ssltls +. auto/compression if [ $NXT_REGEX = YES ]; then . auto/pcre @@ -169,11 +170,13 @@ END NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS $NXT_GNUTLS_CFLAGS \\ $NXT_CYASSL_CFLAGS $NXT_POLARSSL_CFLAGS \\ - $NXT_PCRE_CFLAGS" + $NXT_PCRE_CFLAGS $NXT_ZLIB_CFLAGS $NXT_ZSTD_CFLAGS \\ + $NXT_BROTLI_CFLAGS" NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS $NXT_GNUTLS_LIBS \\ $NXT_CYASSL_LIBS $NXT_POLARSSL_LIBS \\ - $NXT_PCRE_LIB" + $NXT_PCRE_LIB $NXT_ZLIB_LIBS $NXT_ZSTD_LIBS \\ + $NXT_BROTLI_LIBS" if [ $NXT_NJS != NO ]; then . auto/njs diff --git a/src/nxt_brotli.c b/src/nxt_brotli.c new file mode 100644 index 000000000..9ca79bec2 --- /dev/null +++ b/src/nxt_brotli.c @@ -0,0 +1,94 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#define _GNU_SOURCE +#include + + +#include +#include +#include + +#include + +#include "nxt_http_compression.h" + + +static int +nxt_brotli_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + BrotliEncoderState **brotli = &ctx->brotli_ctx; + + *brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (*brotli == NULL) { + return -1; + } + BrotliEncoderSetParameter(*brotli, BROTLI_PARAM_QUALITY, ctx->level); + + printf("%7d %s: brotli compression level [%d]\n", gettid(), __func__, + ctx->level); + + return 0; +} + + +static size_t +nxt_brotli_bound(const nxt_http_comp_compressor_ctx_t *ctx, size_t in_len) +{ + return BrotliEncoderMaxCompressedSize(in_len); +} + + +static ssize_t +nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, + size_t in_len, uint8_t *out_buf, size_t out_len, bool last) +{ + bool ok; + size_t out_bytes = out_len; + BrotliEncoderState *brotli = ctx->brotli_ctx; + + printf("%7d %s: last/%s\n", gettid(), __func__, last ? "true" : "false"); + printf("%7d %s: in_len [%lu] out_len [%lu]\n", gettid(), __func__, + in_len, out_len); + + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_PROCESS, + &in_len, &in_buf, &out_bytes, &out_buf, + NULL); + if (!ok) { + return -1; + } + + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FLUSH, + &in_len, &in_buf, &out_bytes, &out_buf, + NULL); + if (!ok) { + return -1; + } + + printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(), + __func__, in_len, out_len, out_bytes); + if (last) { + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FINISH, + &in_len, &in_buf, &out_bytes, + &out_buf, NULL); + if (!ok) { + return -1; + } + + BrotliEncoderDestroyInstance(brotli); + } + + printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(), + __func__, in_len, out_len, out_bytes); + + return out_len - out_bytes; +} + + +const nxt_http_comp_operations_t nxt_comp_brotli_ops = { + .init = nxt_brotli_init, + .bound = nxt_brotli_bound, + .deflate = nxt_brotli_compress, +}; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index a0b4992fb..8324a4419 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -140,6 +141,12 @@ static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_compression(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); +static nxt_int_t nxt_conf_vldt_compression_encoding(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes_member(nxt_conf_validation_t *vldt, @@ -263,6 +270,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_otel_members[]; #endif static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_compression_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_compressor_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_forwarded_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_client_ip_members[]; #if (NXT_TLS) @@ -403,6 +412,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("chunked_transform"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("compression"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_compression_members, }, NXT_CONF_VLDT_END @@ -465,6 +479,39 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { }; +static nxt_conf_vldt_object_t nxt_conf_vldt_compression_members[] = { + { + .name = nxt_string("types"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("compressors"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_compressors, + }, + + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_compressor_members[] = { + { + .name = nxt_string("encoding"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + .validator = nxt_conf_vldt_compression_encoding, + }, { + .name = nxt_string("level"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("min_length"), + .type = NXT_CONF_VLDT_INTEGER, + }, + + NXT_CONF_VLDT_END +}; + + static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { .name = nxt_string("pass"), @@ -2270,6 +2317,52 @@ nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, } +static nxt_int_t +nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + if (nxt_conf_type(value) == NXT_CONF_ARRAY) { + return nxt_conf_vldt_array_iterator(vldt, value, + &nxt_conf_vldt_compression); + } + + /* NXT_CONF_OBJECT */ + + return nxt_conf_vldt_object_iterator(vldt, value, + &nxt_conf_vldt_compressor_members); +} + + +static nxt_int_t +nxt_conf_vldt_compression(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) +{ + if (nxt_conf_type(value) != NXT_CONF_OBJECT) { + return nxt_conf_vldt_error(vldt, + "The \"compressors\" array must contain " + "only object values."); + } + + return nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_compressor_members); +} + + +static nxt_int_t +nxt_conf_vldt_compression_encoding(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + nxt_str_t token; + + nxt_conf_get_string(value, &token); + + if (nxt_http_comp_compressor_is_valid(&token)) { + return NXT_OK; + } + + return nxt_conf_vldt_error(vldt, "\"%V\" is not a supported compressor.", + &token); +} + + static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_http.h b/src/nxt_http.h index 19bbdda34..79a0ad962 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -33,6 +33,7 @@ typedef enum { NXT_HTTP_FORBIDDEN = 403, NXT_HTTP_NOT_FOUND = 404, NXT_HTTP_METHOD_NOT_ALLOWED = 405, + NXT_HTTP_NOT_ACCEPTABLE = 406, NXT_HTTP_REQUEST_TIMEOUT = 408, NXT_HTTP_LENGTH_REQUIRED = 411, NXT_HTTP_PAYLOAD_TOO_LARGE = 413, @@ -111,6 +112,7 @@ typedef struct { nxt_http_field_t *content_type; nxt_http_field_t *content_length; nxt_off_t content_length_n; + const nxt_str_t *mime_type; } nxt_http_response_t; diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c new file mode 100644 index 000000000..f70a389dd --- /dev/null +++ b/src/nxt_http_compression.c @@ -0,0 +1,837 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + + +#define NXT_COMP_LEVEL_UNSET INT8_MIN + + +typedef enum nxt_http_comp_scheme_e nxt_http_comp_scheme_t; +typedef struct nxt_http_comp_type_s nxt_http_comp_type_t; +typedef struct nxt_http_comp_opts_s nxt_http_comp_opts_t; +typedef struct nxt_http_comp_compressor_s nxt_http_comp_compressor_t; +typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; + +enum nxt_http_comp_scheme_e { + NXT_HTTP_COMP_SCHEME_IDENTITY = 0, +#if NXT_HAVE_ZLIB + NXT_HTTP_COMP_SCHEME_DEFLATE, + NXT_HTTP_COMP_SCHEME_GZIP, +#endif +#if NXT_HAVE_ZSTD + NXT_HTTP_COMP_SCHEME_ZSTD, +#endif +#if NXT_HAVE_BROTLI + NXT_HTTP_COMP_SCHEME_BROTLI, +#endif + + /* keep last */ + NXT_HTTP_COMP_SCHEME_UNKNOWN +}; +#define NXT_NR_COMPRESSORS NXT_HTTP_COMP_SCHEME_UNKNOWN + +struct nxt_http_comp_type_s { + nxt_str_t token; + nxt_http_comp_scheme_t scheme; + int8_t def_compr; + int8_t comp_min; + int8_t comp_max; + + const nxt_http_comp_operations_t *cops; +}; + +struct nxt_http_comp_opts_s { + int8_t level; + nxt_off_t min_len; +}; + +struct nxt_http_comp_compressor_s { + const nxt_http_comp_type_t *type; + nxt_http_comp_opts_t opts; +}; + +struct nxt_http_comp_ctx_s { + nxt_uint_t idx; + + nxt_off_t resp_clen; + + nxt_http_comp_compressor_ctx_t ctx; +}; + + +static nxt_tstr_t *nxt_http_comp_accept_encoding_query; +static nxt_http_route_rule_t *nxt_http_comp_mime_types_rule; +static nxt_http_comp_compressor_t *nxt_http_comp_enabled_compressors; +static nxt_uint_t nxt_http_comp_nr_enabled_compressors; + +static nxt_thread_declare_data(nxt_http_comp_ctx_t, + nxt_http_comp_compressor_ctx); + +#define nxt_http_comp_ctx() nxt_thread_get_data(nxt_http_comp_compressor_ctx) + +static const nxt_conf_map_t nxt_http_comp_compressors_opts_map[] = { + { + nxt_string("level"), + NXT_CONF_MAP_INT, + offsetof(nxt_http_comp_opts_t, level), + }, { + nxt_string("min_length"), + NXT_CONF_MAP_SIZE, + offsetof(nxt_http_comp_opts_t, min_len), + }, +}; + +static const nxt_http_comp_type_t nxt_http_comp_compressors[] = { + /* Keep this first */ + { + .token = nxt_string("identity"), + .scheme = NXT_HTTP_COMP_SCHEME_IDENTITY, +#if NXT_HAVE_ZLIB + }, { + .token = nxt_string("deflate"), + .scheme = NXT_HTTP_COMP_SCHEME_DEFLATE, + .def_compr = NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL, + .comp_min = NXT_HTTP_COMP_ZLIB_COMP_MIN, + .comp_max = NXT_HTTP_COMP_ZLIB_COMP_MAX, + .cops = &nxt_comp_deflate_ops, + }, { + .token = nxt_string("gzip"), + .scheme = NXT_HTTP_COMP_SCHEME_GZIP, + .def_compr = NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL, + .comp_min = NXT_HTTP_COMP_ZLIB_COMP_MIN, + .comp_max = NXT_HTTP_COMP_ZLIB_COMP_MAX, + .cops = &nxt_comp_gzip_ops, +#endif +#if NXT_HAVE_ZSTD + }, { + .token = nxt_string("zstd"), + .scheme = NXT_HTTP_COMP_SCHEME_ZSTD, + .def_compr = NXT_HTTP_COMP_ZSTD_DEFAULT_LEVEL, + .comp_min = NXT_HTTP_COMP_ZSTD_COMP_MIN, + .comp_max = NXT_HTTP_COMP_ZSTD_COMP_MAX, + .cops = &nxt_comp_zstd_ops, +#endif +#if NXT_HAVE_BROTLI + }, { + .token = nxt_string("br"), + .scheme = NXT_HTTP_COMP_SCHEME_BROTLI, + .def_compr = NXT_HTTP_COMP_BROTLI_DEFAULT_LEVEL, + .comp_min = NXT_HTTP_COMP_BROTLI_COMP_MIN, + .comp_max = NXT_HTTP_COMP_BROTLI_COMP_MAX, + .cops = &nxt_comp_brotli_ops, +#endif + }, +}; + + +static void print_compressor(const nxt_http_comp_compressor_t *c) +{ + printf("token : %s\n", c->type->token.start); + printf("scheme : %d\n", c->type->scheme); + printf("level : %d\n", c->opts.level); + printf("min_len : %ld\n", c->opts.min_len); +} + +static void print_comp_config(size_t n) +{ + for (size_t i = 0; i < n; i++) { + nxt_http_comp_compressor_t *compr = + nxt_http_comp_enabled_compressors + i; + + print_compressor(compr); + printf("\n"); + } +} + + +static ssize_t +nxt_http_comp_compress(uint8_t *dst, size_t dst_size, const uint8_t *src, + size_t src_size, bool last) +{ + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_compressor_t *compressor; + const nxt_http_comp_operations_t *cops; + + compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; + cops = compressor->type->cops; + + return cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); +} + + +static size_t +nxt_http_comp_bound(size_t size) +{ + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_compressor_t *compressor; + const nxt_http_comp_operations_t *cops; + + compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; + cops = compressor->type->cops; + + return cops->bound(&ctx->ctx, size); +} + + +nxt_int_t +nxt_http_comp_compress_app_response(nxt_http_request_t *r, nxt_buf_t **b) +{ + bool last; + size_t buf_len, in_len; + ssize_t cbytes; +// uint8_t *buf; +// nxt_buf_t *buf; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + + printf("%s: \n", __func__); + + if (ctx->idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + printf("%s: NXT_HTTP_COMP_SCHEME_IDENTITY [skipping/identity]\n", + __func__); + return NXT_OK; + } + + if ((*b)->mem.pos == NULL) { + return NXT_OK; + } + + last = !(*b)->next || nxt_buf_is_last(*b); + + in_len = (*b)->mem.free - (*b)->mem.pos; + buf_len = nxt_http_comp_bound(in_len); + + printf("%s: in_len [%lu] buf_len [%lu] last [%s]\n", __func__, + in_len, buf_len, last ? "true" : "false"); + +#if 1 + if (buf_len > (size_t)nxt_buf_mem_size(&(*b)->mem)) { + /* XXX Un-skip Content-Length header, or not... */ + return NXT_OK; + } + + uint8_t *buf = nxt_malloc(buf_len); + + cbytes = nxt_http_comp_compress(buf, buf_len, (*b)->mem.pos, in_len, last); + printf("%s: cbytes = %ld\n", __func__, cbytes); + if (cbytes == -1) { + nxt_free(buf); + return NXT_ERROR; + } + + (*b)->mem.free = nxt_cpymem((*b)->mem.pos, buf, cbytes); + + nxt_free(buf); +#else + /* + * While this produces correct compressed output, the router + * process then crashes doing some shutdown/cleanup work. + * + * My best guess as to why this happens is due to our *new* b + * not being allocated from the same memory pool as the original. + */ + nxt_buf_t *buf = nxt_buf_mem_alloc(r->mem_pool, buf_len, 0); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(buf, *b, offsetof(nxt_buf_t, mem)); + + cbytes = nxt_http_comp_compress(buf->mem.start, buf_len, + (*b)->mem.pos, in_len, last); + printf("%s: cbytes = %ld\n", __func__, cbytes); + if (cbytes == -1) { + return NXT_ERROR; + } + + buf->mem.free += cbytes; + /* Seemingly *not* the memory ppol b was allocated from... */ +// nxt_buf_free(r->mem_pool, *b); + *b = buf; + + printf("%s: buf [%p] *b [%p]\n", __func__, buf, *b); +#endif + + return NXT_OK; +} + + +nxt_int_t +nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_file_t **f, + nxt_file_info_t *fi, + size_t static_buf_len, + size_t *out_total) +{ + char tmp_path[NXT_MAX_PATH_LEN]; + size_t in_size, out_size, rest; + u_char *p; + uint8_t *in, *out; + nxt_int_t ret; + nxt_file_t tfile; + nxt_runtime_t *rt = task->thread->runtime; + + static const char *template = "unit-compr-XXXXXX"; + + *out_total = 0; + + if (nxt_slow_path(strlen(rt->tmp) + 1 + strlen(template) + 1 + > NXT_MAX_PATH_LEN)) + { + return NXT_ERROR; + } + + p = nxt_cpymem(tmp_path, rt->tmp, strlen(rt->tmp)); + *p++ = '/'; + p = nxt_cpymem(p, template, strlen(template)); + *p = '\0'; + + tfile.fd = mkstemp(tmp_path); + if (nxt_slow_path(tfile.fd == -1)) { + nxt_alert(task, "mkstemp(%s) failed %E", tmp_path, nxt_errno); + return NXT_ERROR; + } + unlink(tmp_path); + + in_size = nxt_file_size(fi); + out_size = nxt_http_comp_bound(in_size); + + ret = ftruncate(tfile.fd, out_size); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "ftruncate(%d<%s>, %uz) failed %E", + tfile.fd, tmp_path, out_size, nxt_errno); + nxt_file_close(task, &tfile); + return NXT_ERROR; + } + + in = nxt_mem_mmap(NULL, in_size, PROT_READ, MAP_SHARED, (*f)->fd, 0); + if (nxt_slow_path(in == MAP_FAILED)) { + nxt_file_close(task, &tfile); + return NXT_ERROR; + } + + out = nxt_mem_mmap(NULL, out_size, PROT_READ|PROT_WRITE, MAP_SHARED, + tfile.fd, 0); + if (nxt_slow_path(out == MAP_FAILED)) { + nxt_mem_munmap(in, in_size); + nxt_file_close(task, &tfile); + return NXT_ERROR; + } + + rest = in_size; + + do { + bool last; + size_t n; + ssize_t cbytes; + + n = nxt_min(rest, static_buf_len); + + last = n == rest; + + printf("%s: out_off [%ld] in_off [%ld] last [%s]\n", + __func__, *out_total, in_size - rest, last ? "true" : "false"); + + cbytes = nxt_http_comp_compress(out + *out_total, out_size - *out_total, + in + in_size - rest, n, last); + printf("%s: cbytes [%ld]\n", __func__, cbytes); + + *out_total += cbytes; + rest -= n; + } while (rest > 0); + + nxt_mem_munmap(in, in_size); + msync(out, out_size, MS_ASYNC); + nxt_mem_munmap(out, out_size); + + ret = ftruncate(tfile.fd, *out_total); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "ftruncate(%d<%s>, %uz) failed %E", + tfile.fd, tmp_path, *out_total, nxt_errno); + nxt_file_close(task, &tfile); + return NXT_ERROR; + } + + nxt_file_close(task, *f); + + **f = tfile; + + return NXT_OK; +} + + +bool +nxt_http_comp_wants_compression(void) +{ + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + + printf("%s: compression [%s]\n", __func__, ctx->idx > 0 ? "true" : "false"); + return ctx->idx; +} + + +static nxt_uint_t +nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) +{ + if (token->start[0] == '*') { + return NXT_HTTP_COMP_SCHEME_IDENTITY; + } + + for (nxt_uint_t i = 0; i < nxt_http_comp_nr_enabled_compressors; i++) { + if (nxt_strstr_eq(token, + &nxt_http_comp_enabled_compressors[i].type->token)) + { + return i; + } + } + + return NXT_HTTP_COMP_SCHEME_UNKNOWN; +} + + +/* + * We need to parse the 'Accept-Encoding` header as described by + * + * which can take forms such as + * + * Accept-Encoding: compress, gzip + * Accept-Encoding: + * Accept-Encoding: * + * Accept-Encoding: compress;q=0.5, gzip;q=1.0 + * Accept-Encoding: gzip;q=1.0, identity;q=0.5, *;q=0 + * + * '*:q=0' means if the content being served has no 'Content-Coding' + * matching an 'Accept-Encoding' entry then don't send any response. + * + * 'identity;q=0' seems to basically mean the same thing... + */ +static nxt_int_t +nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) +{ + bool identity_allowed = true; + char *str, *tkn, *tail, *cur; + double weight = 0.0; + nxt_int_t idx = NXT_HTTP_COMP_SCHEME_IDENTITY; + + str = nxt_str_cstrz(r->mem_pool, token); + if (str == NULL) { + return NXT_HTTP_COMP_SCHEME_IDENTITY; + } + + cur = tail = str; + /* + * To ease parsing the Accept-Encoding header, remove all spaces, + * which hold no semantic meaning. + */ + for (; *cur != '\0'; cur++) { + if (*cur == ' ') { + continue; + } + + *tail++ = *cur; + } + *tail = '\0'; + + while ((tkn = strsep(&str, ","))) { + char *qptr; + double qval = 1.0; + nxt_str_t enc; + nxt_uint_t ecidx; + nxt_http_comp_scheme_t scheme; + + qptr = strstr(tkn, ";q="); + if (qptr != NULL) { + errno = 0; + + qval = strtod(qptr + 3, NULL); + + if (errno == ERANGE || qval < 0.0 || qval > 1.0) { + continue; + } + } + + enc.start = (u_char *)tkn; + enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn); + + ecidx = nxt_http_comp_compressor_lookup_enabled(&enc); + if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { + continue; + } + + scheme = nxt_http_comp_enabled_compressors[ecidx].type->scheme; + + printf("%s: %.*s [%f] [%d:%d]\n", __func__, (int)enc.length, enc.start, + qval, ecidx, scheme); + + if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { + identity_allowed = false; + } + + if (qval == 0.0 || qval < weight) { + continue; + } + + idx = ecidx; + weight = qval; + } + + printf("%s: Selected compressor : %s\n", __func__, + nxt_http_comp_enabled_compressors[idx].type->token.start); + + printf("%s: idx [%u], identity_allowed [%s]\n", __func__, idx, + identity_allowed ? "true" : "false"); + + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) { + return -1; + } + + return idx; +} + + +static nxt_int_t +nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx) +{ + const nxt_str_t *token; + nxt_http_field_t *f; + + static const nxt_str_t content_encoding_str = + nxt_string("Content-Encoding"); + + printf("%s: \n", __func__); + +#if 0 + if (comp_idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + return NXT_OK; + } +#endif + + f = nxt_list_add(r->resp.fields); + if (nxt_slow_path(f == NULL)) { + return NXT_ERROR; + } + + token = &nxt_http_comp_enabled_compressors[comp_idx].type->token; + + *f = (nxt_http_field_t){}; + + f->name = content_encoding_str.start; + f->name_length = content_encoding_str.length; + f->value = token->start; + f->value_length = token->length; + + r->resp.content_length = NULL; + r->resp.content_length_n = -1; + + if (r->resp.mime_type == NULL) { + nxt_http_field_t *f; + + /* + * As per RFC 2616 section 4.4 item 3, you should not send + * Content-Length when a Transfer-Encoding header is present. + */ + nxt_list_each(f, r->resp.fields) { + if (nxt_strcasecmp(f->name, + (const u_char *)"Content-Length") == 0) + { + printf("%s: Found (%s: %s), marking as 'skip'\n", __func__, + f->name, f->value); + f->skip = true; + break; + } + } nxt_list_loop; + } + + return NXT_OK; +} + + +static bool +nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) +{ + nxt_http_field_t *f; + + printf("%s: \n", __func__); + + nxt_list_each(f, r->resp.fields) { + printf("%s: %.*s: %.*s\n", __func__, f->name_length, f->name, + f->value_length, f->value); + if (nxt_strcasecmp(f->name, (const u_char *)"Content-Encoding") == 0) { + return true; + } + } nxt_list_loop; + + return false; +} + + +nxt_int_t +nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) +{ + int err; + nxt_int_t ret, idx; + nxt_off_t min_len; + nxt_str_t accept_encoding, mime_type = {}; + nxt_router_conf_t *rtcf; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_compressor_t *compressor; + + printf("%s: \n", __func__); + + *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1 }; + + if (nxt_http_comp_nr_enabled_compressors == 0) { + return NXT_OK; + } + + if (r->resp.content_length_n == 0) { + return NXT_OK; + } + + if (r->resp.mime_type != NULL) { + mime_type = *r->resp.mime_type; + } else if (r->resp.content_type != NULL) { + mime_type.start = r->resp.content_type->value; + mime_type.length = r->resp.content_type->value_length; + } + + if (mime_type.start == NULL) { + return NXT_OK; + } + + printf("%s: Response Content-Type [%.*s]\n", __func__, + (int)mime_type.length, mime_type.start); + + if (nxt_http_comp_mime_types_rule != NULL) { + ret = nxt_http_route_test_rule(r, nxt_http_comp_mime_types_rule, + mime_type.start, + mime_type.length); + printf("%s: mime_type : %d (%.*s)\n", __func__, ret, + (int)mime_type.length, mime_type.start); + if (ret == 0) { + return NXT_OK; + } + } + + rtcf = r->conf->socket_conf->router_conf; + + if (nxt_http_comp_is_resp_content_encoded(r)) { + return NXT_OK; + } + + /* XXX Should checking the Accept-Encoding header come first? */ + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, + r, r->mem_pool); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + + ret = nxt_tstr_query(task, r->tstr_query, + nxt_http_comp_accept_encoding_query, &accept_encoding); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + idx = nxt_http_comp_select_compressor(r, &accept_encoding); + if (idx == -1) { + return NXT_HTTP_NOT_ACCEPTABLE; + } + + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + return NXT_OK; + } + + compressor = &nxt_http_comp_enabled_compressors[idx]; + + if (r->resp.content_length_n > -1) { + ctx->resp_clen = r->resp.content_length_n; + } else if (r->resp.content_length != NULL) { + ctx->resp_clen = + strtol((char *)r->resp.content_length->value, NULL, 10); + } + + min_len = compressor->opts.min_len; + + printf("%s: content_length [%ld] min_len [%ld]\n", __func__, + ctx->resp_clen, min_len); + if (ctx->resp_clen > -1 && ctx->resp_clen < min_len) { + printf("%s: %ld < %ld [skipping/clen]\n", __func__, + ctx->resp_clen, min_len); + return NXT_OK; + } + + nxt_http_comp_set_header(r, idx); + + ctx->idx = idx; + ctx->ctx.level = nxt_http_comp_enabled_compressors[idx].opts.level; + + err = compressor->type->cops->init(&ctx->ctx); + if (nxt_slow_path(err)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_uint_t +nxt_http_comp_compressor_token2idx(const nxt_str_t *token) +{ + for (nxt_uint_t i = 0; i < nxt_nitems(nxt_http_comp_compressors); i++) { + if (nxt_strstr_eq(token, &nxt_http_comp_compressors[i].token)) { + return i; + } + } + + return NXT_HTTP_COMP_SCHEME_UNKNOWN; +} + + +bool +nxt_http_comp_compressor_is_valid(const nxt_str_t *token) +{ + nxt_uint_t idx; + + idx = nxt_http_comp_compressor_token2idx(token); + if (idx != NXT_HTTP_COMP_SCHEME_UNKNOWN) { + return true; + } + + return false; +} + + +static nxt_int_t +nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf, + const nxt_conf_value_t *comp, nxt_uint_t index) +{ + nxt_int_t ret; + nxt_str_t token; + nxt_uint_t cidx; + nxt_conf_value_t *obj; + nxt_http_comp_compressor_t *compr; + + static const nxt_str_t token_str = nxt_string("encoding"); + + printf("%s: \n", __func__); + + obj = nxt_conf_get_object_member(comp, &token_str, NULL); + if (obj == NULL) { + return NXT_ERROR; + } + + nxt_conf_get_string(obj, &token); + cidx = nxt_http_comp_compressor_token2idx(&token); + + compr = &nxt_http_comp_enabled_compressors[index]; + + compr->type = &nxt_http_comp_compressors[cidx]; + compr->opts.level = compr->type->def_compr; + compr->opts.min_len = -1; + printf("%s: %s\n", __func__, compr->type->token.start); + + ret = nxt_conf_map_object(rtcf->mem_pool, comp, + nxt_http_comp_compressors_opts_map, + nxt_nitems(nxt_http_comp_compressors_opts_map), + &compr->opts); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + + if (compr->opts.level < compr->type->comp_min + || compr->opts.level > compr->type->comp_max) + { + nxt_log(task, NXT_LOG_NOTICE, + "Overriding invalid compression level for [%V] [%d] -> [%d]", + &compr->type->token, compr->opts.level, + compr->type->def_compr); + compr->opts.level = compr->type->def_compr; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, + const nxt_conf_value_t *comp_conf) +{ + nxt_int_t ret; + nxt_uint_t n = 1; /* 'identity' */ + nxt_conf_value_t *comps, *mimes; + + static const nxt_str_t accept_enc_str = + nxt_string("$header_accept_encoding"); + static const nxt_str_t comps_str = nxt_string("compressors"); + static const nxt_str_t mimes_str = nxt_string("types"); + + printf("%s: \n", __func__); + + mimes = nxt_conf_get_object_member(comp_conf, &mimes_str, NULL); + if (mimes != NULL) { + nxt_http_comp_mime_types_rule = + nxt_http_route_types_rule_create(task, + rtcf->mem_pool, mimes); + if (nxt_slow_path(nxt_http_comp_mime_types_rule == NULL)) { + return NXT_ERROR; + } + } + + nxt_http_comp_accept_encoding_query = + nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str, + NXT_TSTR_STRZ); + if (nxt_slow_path(nxt_http_comp_accept_encoding_query == NULL)) { + return NXT_ERROR; + } + + comps = nxt_conf_get_object_member(comp_conf, &comps_str, NULL); + if (nxt_slow_path(comps == NULL)) { + return NXT_ERROR; + } + + if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { + n++; + } else { + n += nxt_conf_object_members_count(comps); + } + nxt_http_comp_nr_enabled_compressors = n; + + nxt_http_comp_enabled_compressors = + nxt_mp_zalloc(rtcf->mem_pool, + sizeof(nxt_http_comp_compressor_t) * n); + + nxt_http_comp_enabled_compressors[0] = + (nxt_http_comp_compressor_t){ .type = &nxt_http_comp_compressors[0], + .opts.level = NXT_COMP_LEVEL_UNSET, + .opts.min_len = -1 }; + + if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { + print_comp_config(nxt_http_comp_nr_enabled_compressors); + return nxt_http_comp_set_compressor(task, rtcf, comps, 1); + } + + for (nxt_uint_t i = 1; i < nxt_http_comp_nr_enabled_compressors; i++) { + nxt_conf_value_t *obj; + + obj = nxt_conf_get_array_element(comps, i - 1); + ret = nxt_http_comp_set_compressor(task, rtcf, obj, i); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } + } + + print_comp_config(nxt_http_comp_nr_enabled_compressors); + + return NXT_OK; +} diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h new file mode 100644 index 000000000..a9e7af7bc --- /dev/null +++ b/src/nxt_http_compression.h @@ -0,0 +1,105 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#ifndef _NXT_COMPRESSION_H_INCLUDED_ +#define _NXT_COMPRESSION_H_INCLUDED_ + +#include + +#include +#include +#include + +#if NXT_HAVE_ZLIB +#include +#endif + +#if NXT_HAVE_ZSTD +#include +#endif + +#if NXT_HAVE_BROTLI +#include +#endif + +#include +#include +#include +#include + + +#if NXT_HAVE_ZLIB +#define NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL Z_DEFAULT_COMPRESSION +#define NXT_HTTP_COMP_ZLIB_COMP_MIN Z_DEFAULT_COMPRESSION +#define NXT_HTTP_COMP_ZLIB_COMP_MAX Z_BEST_COMPRESSION +#endif +#if NXT_HAVE_ZSTD +#define NXT_HTTP_COMP_ZSTD_DEFAULT_LEVEL ZSTD_CLEVEL_DEFAULT +#define NXT_HTTP_COMP_ZSTD_COMP_MIN (-7) +#define NXT_HTTP_COMP_ZSTD_COMP_MAX 22 +#endif +#if NXT_HAVE_BROTLI +#define NXT_HTTP_COMP_BROTLI_DEFAULT_LEVEL BROTLI_DEFAULT_QUALITY +#define NXT_HTTP_COMP_BROTLI_COMP_MIN BROTLI_MIN_QUALITY +#define NXT_HTTP_COMP_BROTLI_COMP_MAX BROTLI_MAX_QUALITY +#endif + + +typedef struct nxt_http_comp_compressor_ctx_s nxt_http_comp_compressor_ctx_t; +typedef struct nxt_http_comp_operations_s nxt_http_comp_operations_t; + +struct nxt_http_comp_compressor_ctx_s { + int8_t level; + + union { +#if NXT_HAVE_ZLIB + z_stream zlib_ctx; +#endif +#if NXT_HAVE_ZSTD + ZSTD_CStream *zstd_ctx; +#endif +#if NXT_HAVE_BROTLI + BrotliEncoderState *brotli_ctx; +#endif + }; +}; + +struct nxt_http_comp_operations_s { + int (*init)(nxt_http_comp_compressor_ctx_t *ctx); + size_t (*bound)(const nxt_http_comp_compressor_ctx_t *ctx, + size_t in_len); + ssize_t (*deflate)(nxt_http_comp_compressor_ctx_t *ctx, + const uint8_t *in_buf, size_t in_len, + uint8_t *out_buf, size_t out_len, bool last); +}; + + +#if NXT_HAVE_ZLIB +extern const nxt_http_comp_operations_t nxt_comp_deflate_ops; +extern const nxt_http_comp_operations_t nxt_comp_gzip_ops; +#endif + +#if NXT_HAVE_ZSTD +extern const nxt_http_comp_operations_t nxt_comp_zstd_ops; +#endif + +#if NXT_HAVE_BROTLI +extern const nxt_http_comp_operations_t nxt_comp_brotli_ops; +#endif + + +extern nxt_int_t nxt_http_comp_compress_app_response(nxt_http_request_t *r, + nxt_buf_t **b); +extern nxt_int_t nxt_http_comp_compress_static_response(nxt_task_t *task, + nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, + size_t *out_total); +extern bool nxt_http_comp_wants_compression(void); +extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); +extern nxt_int_t nxt_http_comp_check_compression(nxt_task_t *task, + nxt_http_request_t *r); +extern nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, + nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf); + +#endif /* _NXT_COMPRESSION_H_INCLUDED_ */ diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 722197b8a..1ecefcd45 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -320,6 +320,8 @@ nxt_http_request_start(nxt_task_t *task, void *obj, void *data) nxt_socket_conf_t *skcf; nxt_http_request_t *r; + printf("%s: \n", __func__); + r = obj; NXT_OTEL_TRACE(); @@ -692,6 +694,8 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_field_t *server, *date, *content_length; nxt_socket_conf_t *skcf; + printf("%s: \n", __func__); + ret = nxt_http_set_headers(r); if (nxt_slow_path(ret != NXT_OK)) { goto fail; @@ -783,6 +787,9 @@ void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { if (nxt_fast_path(r->proto.any != NULL)) { + printf("%s: sending [%lu] bytes\n", __func__, + nxt_buf_mem_used_size(&out->mem)); + nxt_http_proto[r->protocol].send(task, r, out); } } diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index bd0646f3b..e08fc9866 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -688,6 +688,8 @@ nxt_http_action_init(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_router_conf_t *rtcf; nxt_http_action_conf_t acf; + printf("%s: \n", __func__); + nxt_memzero(&acf, sizeof(acf)); ret = nxt_conf_map_object(tmcf->mem_pool, cv, nxt_http_route_action_conf, diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index d56ec5870..2ad919271 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -5,6 +5,7 @@ #include #include +#include typedef struct { @@ -326,6 +327,8 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler; nxt_http_static_conf_t *conf; + printf("%s: \n", __func__); + action = ctx->action; conf = action->u.conf; rtcf = r->conf->socket_conf->router_conf; @@ -576,7 +579,37 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, field->value_length = mtype->length; } + r->resp.mime_type = mtype; + if (ctx->need_body && nxt_file_size(&fi) > 0) { + ret = nxt_http_comp_check_compression(task, r); + if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); + return; + } else if (ret != NXT_OK) { + goto fail; + } + + if (nxt_http_comp_wants_compression()) { + size_t out_total; + nxt_int_t ret; + + ret = nxt_http_comp_compress_static_response( + task, &f, &fi, + NXT_HTTP_STATIC_BUF_SIZE, + &out_total); + if (ret == NXT_ERROR) { + goto fail; + } + + ret = nxt_file_info(f, &fi); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + r->resp.content_length_n = out_total; + } + fb = nxt_mp_zget(r->mem_pool, NXT_BUF_FILE_SIZE); if (nxt_slow_path(fb == NULL)) { goto fail; @@ -793,6 +826,8 @@ nxt_http_static_body_handler(nxt_task_t *task, void *obj, void *data) nxt_work_queue_t *wq; nxt_http_request_t *r; + printf("%s: \n", __func__); + r = obj; fb = r->out; @@ -853,6 +888,8 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) nxt_off_t rest; nxt_http_request_t *r; + printf("%s: \n", __func__); + b = obj; r = data; diff --git a/src/nxt_router.c b/src/nxt_router.c index 44ea823b7..28c0888f9 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -21,6 +21,7 @@ #include #include #include +#include #define NXT_SHARED_PORT_ID 0xFFFFu @@ -1680,6 +1681,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, static const nxt_str_t static_path = nxt_string("/settings/http/static"); static const nxt_str_t websocket_path = nxt_string("/settings/http/websocket"); + static const nxt_str_t compression_path = + nxt_string("/settings/http/compression"); static const nxt_str_t forwarded_path = nxt_string("/forwarded"); static const nxt_str_t client_ip_path = nxt_string("/client_ip"); #if (NXT_HAVE_OTEL) @@ -2044,6 +2047,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_str_null(&skcf->body_temp_path); if (http != NULL) { + nxt_conf_value_t *comp; + ret = nxt_conf_map_object(mp, http, nxt_router_http_conf, nxt_nitems(nxt_router_http_conf), skcf); @@ -2051,6 +2056,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_alert(task, "http map error"); goto fail; } + + comp = nxt_conf_get_path(root, &compression_path); + if (comp != NULL) { + nxt_http_comp_compression_init(task, rtcf, comp); + } } if (websocket != NULL) { @@ -4126,6 +4136,8 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_unit_response_t *resp; nxt_request_rpc_data_t *req_rpc_data; + printf("%s: \n", __func__); + req_rpc_data = data; r = req_rpc_data->request; @@ -4180,9 +4192,10 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, } if (r->header_sent) { + nxt_http_comp_compress_app_response(r, &b); + nxt_buf_chain_add(&r->out, b); nxt_http_request_send_body(task, r, NULL); - } else { b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; @@ -4262,6 +4275,11 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_buf_chain_add(&r->out, b); } + ret = nxt_http_comp_check_compression(task, r); + if (ret != NXT_OK) { + goto fail; + } + nxt_http_request_header_send(task, r, nxt_http_request_send_body, NULL); if (r->websocket_handshake diff --git a/src/nxt_zlib.c b/src/nxt_zlib.c new file mode 100644 index 000000000..fffb9b43b --- /dev/null +++ b/src/nxt_zlib.c @@ -0,0 +1,90 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#include +#include +#include + +#include + +#include "nxt_http_compression.h" + + +static int +nxt_zlib_gzip_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + z_stream *z = &ctx->zlib_ctx; + + *z = (z_stream){}; + + return deflateInit2(z, ctx->level, Z_DEFLATED, 9 + 16, 8, + Z_DEFAULT_STRATEGY); +} + + +static int +nxt_zlib_deflate_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + z_stream *z = &ctx->zlib_ctx; + + *z = (z_stream){}; + + return deflateInit2(z, ctx->level, Z_DEFLATED, 9, 8, Z_DEFAULT_STRATEGY); +} + + +static size_t +nxt_zlib_bound(const nxt_http_comp_compressor_ctx_t *ctx, size_t in_len) +{ + z_stream *z = (z_stream *)&ctx->zlib_ctx; + + return deflateBound(z, in_len); +} + + +static ssize_t +nxt_zlib_deflate(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, + size_t in_len, uint8_t *out_buf, size_t out_len, bool last) +{ + int ret; + size_t compressed_bytes; + z_stream *z = &ctx->zlib_ctx; + + z->avail_in = in_len; + z->next_in = (z_const Bytef *)in_buf; + + z->avail_out = out_len; + z->next_out = out_buf; + + printf("%s: in_len [%lu], out_len [%lu]\n", __func__, in_len, out_len); + + compressed_bytes = z->total_out; + + ret = deflate(z, last ? Z_FINISH : Z_SYNC_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR) { + deflateEnd(z); + printf("%s: ret = %d\n", __func__, ret); + return -1; + } + + if (last) + deflateEnd(z); + + return z->total_out - compressed_bytes; +} + + +const nxt_http_comp_operations_t nxt_comp_deflate_ops = { + .init = nxt_zlib_deflate_init, + .bound = nxt_zlib_bound, + .deflate = nxt_zlib_deflate, +}; + + +const nxt_http_comp_operations_t nxt_comp_gzip_ops = { + .init = nxt_zlib_gzip_init, + .bound = nxt_zlib_bound, + .deflate = nxt_zlib_deflate, +}; diff --git a/src/nxt_zstd.c b/src/nxt_zstd.c new file mode 100644 index 000000000..6430a6ca1 --- /dev/null +++ b/src/nxt_zstd.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#include +#include +#include + +#include + +#include "nxt_http_compression.h" + + +static int +nxt_zstd_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + ZSTD_CStream **zstd = &ctx->zstd_ctx; + + *zstd = ZSTD_createCStream(); + if (*zstd == NULL) { + return -1; + } + ZSTD_initCStream(*zstd, ctx->level); + + printf("%s: zstd compression level [%d]\n", __func__, ctx->level); + + return 0; +} + + +static size_t +nxt_zstd_bound(const nxt_http_comp_compressor_ctx_t *ctx, size_t in_len) +{ + return ZSTD_compressBound(in_len); +} + + +static ssize_t +nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, + size_t in_len, uint8_t *out_buf, size_t out_len, bool last) +{ + size_t ret; + ZSTD_CStream *zstd = ctx->zstd_ctx; + ZSTD_inBuffer zinb = { .src = in_buf, .size = in_len }; + ZSTD_outBuffer zoutb = { .dst = out_buf, .size = out_len }; + + printf("%s: in_len [%lu] out_len [%lu] last [%s]\n", __func__, + in_len, out_len, last ? "true" : "false"); + + ret = ZSTD_compressStream(zstd, &zoutb, &zinb); + + if (zinb.pos < zinb.size) { + printf("%s: short by [%d]\n", __func__, zinb.pos < zinb.size); + ret = ZSTD_flushStream(zstd, &zoutb); + } + + if (last) { + ret = ZSTD_endStream(zstd, &zoutb); + ZSTD_freeCStream(zstd); + } + + printf("%s: ret [%lu]\n", __func__, ret); + if (ZSTD_isError(ret)) { + printf("%s: [%s]\n", __func__, ZSTD_getErrorName(ret)); + return -1; + } + + return zoutb.pos; +} + + +const nxt_http_comp_operations_t nxt_comp_zstd_ops = { + .init = nxt_zstd_init, + .bound = nxt_zstd_bound, + .deflate = nxt_zstd_compress, +};