From 4e9414ba6eaeaf889228d17a1fba1ab11b1dd0d7 Mon Sep 17 00:00:00 2001 From: moctardiouf Date: Thu, 12 Dec 2024 11:08:23 +0100 Subject: [PATCH] PHRAS-3588 implement http request quota by type (#4564) * PHRAS-3588 manage http request limits by verbs * PHRAS-3588 change limit method * re-introduce burst parameters * PHRAS-3588 add activation boolean * applying auto-documentation format for env var --- .env | 35 +++++++ docker-compose.yml | 8 ++ docker/nginx/root/entrypoint.sh | 10 +- docker/nginx/root/nginx.conf.sample | 3 +- .../root/nginx.request_limits.conf.sample | 95 +++++++++++++++++++ 5 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 docker/nginx/root/nginx.request_limits.conf.sample diff --git a/.env b/.env index 8acdb77b78..96710b001e 100644 --- a/.env +++ b/.env @@ -224,6 +224,41 @@ GATEWAY_DENIED_IPS= # @run GATEWAY_USERS= + + +# Manage http incoming request limits by verbs +# this feature is based on ip adresses and need PHRASEANET_TRUSTED_PROXIES +# defined to get real_ip +# READ is for GET and HEAD requests +# WRITE is for POST, PUT, DELETE and PATCH requests +# Enabling the requests Limit +# @run +HTTP_REQUEST_LIMITS=false + +# (m) For Exemple 16,000 IP addresses takes 1 megabyte, so our zone can store about 160,000 addresses. +# @run +HTTP_READ_REQUEST_LIMIT_MEMORY=10 + +# (r/s) Sets the maximum request rate. By default here the rate cannot exceed 10 requests per second +# @run +HTTP_READ_REQUEST_LIMIT_RATE=100 + +# The burst parameter defines how many requests a client can make in excess of the rate specified +# @run +HTTP_READ_REQUEST_LIMIT_BURST=20 + +# (m) For Exemple 16,000 IP addresses takes 1 megabyte, so our zone can store about 160,000 addresses. +# @run +HTTP_WRITE_REQUEST_LIMIT_MEMORY=10 + +# (r/s) Sets the maximum request rate. By default here the rate cannot exceed 10 requests per second +# @run +HTTP_WRITE_REQUEST_LIMIT_RATE=100 + +# The burst parameter defines how many requests a client can make in excess of the rate specified +# @run +HTTP_WRITE_REQUEST_LIMIT_BURST=20 + # https and reverse proxy (on/off) # set to on in the case : https behind a proxy # @run diff --git a/docker-compose.yml b/docker-compose.yml index b378c472f7..3337418807 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,14 @@ services: - GATEWAY_DENIED_IPS - GATEWAY_USERS - GATEWAY_CSP + - HTTP_REQUEST_LIMITS + - HTTP_READ_REQUEST_LIMIT_MEMORY + - HTTP_READ_REQUEST_LIMIT_RATE + - HTTP_WRITE_REQUEST_LIMIT_MEMORY + - HTTP_WRITE_REQUEST_LIMIT_RATE + - HTTP_READ_REQUEST_LIMIT_BURST + - HTTP_WRITE_REQUEST_LIMIT_BURST + ports: - ${PHRASEANET_APP_PORT}:80 networks: diff --git a/docker/nginx/root/entrypoint.sh b/docker/nginx/root/entrypoint.sh index d448228402..f9015233ee 100755 --- a/docker/nginx/root/entrypoint.sh +++ b/docker/nginx/root/entrypoint.sh @@ -35,7 +35,15 @@ else envsubst < "/securitycontentpolicies.sample.conf" > /etc/nginx/conf.d/securitycontentpolicies.conf fi -cat /nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g" | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g" | sed "s/\$NEW_TARGET/$NEW_TARGET/g" | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" > /etc/nginx/conf.d/default.conf + +if [[ $HTTP_REQUEST_LIMITS && $HTTP_REQUEST_LIMITS = true ]] && [[ ! -z $HTTP_READ_REQUEST_LIMIT_MEMORY || ! -z $HTTP_READ_REQUEST_LIMIT_RATE || ! -z $HTTP_READ_REQUEST_LIMIT_BURST || ! -z $HTTP_WRITE_REQUEST_LIMIT_MEMORY || ! -z $HTTP_WRITE_REQUEST_LIMIT_RATE || ! -z $HTTP_WRITE_REQUEST_LIMIT_BURST ]]; then + echo "HTTP_REQUEST_LIMITS is $HTTP_REQUEST_LIMITS" + cat /nginx.request_limits.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g" | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g" | sed "s/\$NEW_TARGET/$NEW_TARGET/g" | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" | sed "s/\$HTTP_READ_REQUEST_LIMIT_MEMORY/$HTTP_READ_REQUEST_LIMIT_MEMORY/g" | sed "s/\$HTTP_READ_REQUEST_LIMIT_RATE/$HTTP_READ_REQUEST_LIMIT_RATE/g" | sed "s/\$HTTP_WRITE_REQUEST_LIMIT_MEMORY/$HTTP_WRITE_REQUEST_LIMIT_MEMORY/g" | sed "s/\$HTTP_WRITE_REQUEST_LIMIT_RATE/$HTTP_WRITE_REQUEST_LIMIT_RATE/g" | sed "s/\$HTTP_READ_REQUEST_LIMIT_BURST/$HTTP_READ_REQUEST_LIMIT_BURST/g"| sed "s/\$HTTP_WRITE_REQUEST_LIMIT_BURST/$HTTP_WRITE_REQUEST_LIMIT_BURST/g" > /etc/nginx/conf.d/default.conf +else + echo "HTTP_REQUEST_LIMITS is $HTTP_REQUEST_LIMITS or not defined" + cat /nginx.conf.sample | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_SEND_TIMEOUT/$GATEWAY_SEND_TIMEOUT/g" | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" | sed "s/\$MAX_BODY_SIZE/$MAX_BODY_SIZE/g" | sed "s/\$GATEWAY_PROXY_TIMEOUT/$GATEWAY_PROXY_TIMEOUT/g" | sed "s/\$NEW_TARGET/$NEW_TARGET/g" | sed "s/\$NEW_RESOLVER/$NEW_RESOLVER/g" | sed "s/\$GATEWAY_FASTCGI_HTTPS/$GATEWAY_FASTCGI_HTTPS/g" > /etc/nginx/conf.d/default.conf +fi + cat /fastcgi_timeout.conf | sed "s/\$GATEWAY_FASTCGI_TIMEOUT/$GATEWAY_FASTCGI_TIMEOUT/g" > /etc/nginx/fastcgi_extended_params echo `date +"%Y-%m-%d %H:%M:%S"` " - Setting for real_ip_from using Trusted Proxies" diff --git a/docker/nginx/root/nginx.conf.sample b/docker/nginx/root/nginx.conf.sample index 8fc390b9dd..2d2fc61bcb 100644 --- a/docker/nginx/root/nginx.conf.sample +++ b/docker/nginx/root/nginx.conf.sample @@ -1,3 +1,4 @@ + send_timeout $GATEWAY_SEND_TIMEOUT; keepalive_timeout $GATEWAY_SEND_TIMEOUT; proxy_connect_timeout $GATEWAY_PROXY_TIMEOUT; @@ -5,6 +6,7 @@ proxy_send_timeout $GATEWAY_PROXY_TIMEOUT; client_header_timeout $GATEWAY_SEND_TIMEOUT; client_body_timeout $GATEWAY_SEND_TIMEOUT; fastcgi_read_timeout $GATEWAY_FASTCGI_TIMEOUT; + resolver $NEW_RESOLVER; upstream backend { @@ -36,7 +38,6 @@ server { if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) { return 503; } - # First attempt to serve request as file, then # as directory, then fall back to index.html try_files $uri $uri/ @rewriteapp; diff --git a/docker/nginx/root/nginx.request_limits.conf.sample b/docker/nginx/root/nginx.request_limits.conf.sample new file mode 100644 index 0000000000..a6b1cf3e0f --- /dev/null +++ b/docker/nginx/root/nginx.request_limits.conf.sample @@ -0,0 +1,95 @@ + +send_timeout $GATEWAY_SEND_TIMEOUT; +keepalive_timeout $GATEWAY_SEND_TIMEOUT; +proxy_connect_timeout $GATEWAY_PROXY_TIMEOUT; +proxy_send_timeout $GATEWAY_PROXY_TIMEOUT; +client_header_timeout $GATEWAY_SEND_TIMEOUT; +client_body_timeout $GATEWAY_SEND_TIMEOUT; +fastcgi_read_timeout $GATEWAY_FASTCGI_TIMEOUT; + +map $request_method $postlimit { + default ""; + POST $binary_remote_addr; +} + +map $request_method $getlimit { + default ""; + GET $binary_remote_addr; +} + +limit_req_status 429; +limit_req_zone $getlimit zone=readlimitsbyip:$HTTP_READ_REQUEST_LIMIT_MEMORYm rate=$HTTP_READ_REQUEST_LIMIT_RATEr/s; +limit_req_zone $postlimit zone=writelimitsbyip:$HTTP_WRITE_REQUEST_LIMIT_MEMORYm rate=$HTTP_WRITE_REQUEST_LIMIT_RATEr/s; +resolver $NEW_RESOLVER; + +upstream backend { + server phraseanet:9000; +} + +#upstream samlsp { +# server phraseanet-saml-sp:8080; +#} + +server { + listen 80; + root /var/alchemy/Phraseanet/www; + + index index.php; + client_max_body_size $MAX_BODY_SIZE; + + location /api { + if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) { + return 503; + } + rewrite ^(.*)$ /api.php/$1 last; + } + + location / { + + error_page 503 = @maintenance; + recursive_error_pages on; + if (-f /var/alchemy/Phraseanet/datas/nginx/maintenance.html) { + return 503; + } + # First attempt to serve request as file, then + # as directory, then fall back to index.html + try_files $uri $uri/ @rewriteapp; + limit_req zone=readlimitsbyip burst=$HTTP_READ_REQUEST_LIMIT_BURST nodelay; + limit_req zone=writelimitsbyip burst=$HTTP_WRITE_REQUEST_LIMIT_BURST nodelay; + } + + location @rewriteapp { + rewrite ^(.*)$ /index.php/$1 last; + } + + # PHP scripts -> PHP-FPM server listening on 127.0.0.1:9000 + location ~ ^/(index|index_dev|api)\.php(/|$) { + fastcgi_pass backend; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + include fastcgi_params; + $GATEWAY_FASTCGI_HTTPS + include restrictions; + } + + location ~ ^/(status|ping)$ { + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_index index.php; + include fastcgi_params; + include fastcgi_extended_params; + fastcgi_pass backend; + } + + location /simplesaml/ { + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + set $target $NEW_TARGET:8080; + proxy_pass http://$target; + + } + + location @maintenance { + root /var/alchemy/Phraseanet/datas/nginx/; + try_files $uri /maintenance.html; + } +}