Bellon/getjerry customize branch test

src/autoconf/IngressController.py

@@ -49,13 +49,20 @@ def _to_instances(self, controller_instance) -> List[dict]:
                 instance["env"][env.name] = env.value or ""
         for controller_service in self._get_controller_services():
             if controller_service.metadata.annotations:
+                if "external-dns.alpha.kubernetes.io/hostname" not in controller_service.metadata.annotations:
+                    self._logger.warning(
+                        f"external-dns.alpha.kubernetes.io/hostname not in ingress {controller_service.metadata.name} annotations, Ignoring unsupported ingress.",
+                    )
+                    continue
+                hostname = controller_service.metadata.annotations.get("external-dns.alpha.kubernetes.io/hostname")
                 for (
                     annotation,
                     value,
                 ) in controller_service.metadata.annotations.items():
                     if not annotation.startswith("bunkerweb.io/"):
                         continue
-                    instance["env"][annotation.replace("bunkerweb.io/", "", 1)] = value
+                    config = annotation.replace("bunkerweb.io/", "", 1)
+                    instance["env"][f"{hostname}_{config}"] = value
         return [instance]
 
     def _get_controller_services(self) -> list:
@@ -66,6 +73,8 @@ def _to_services(self, controller_service) -> List[dict]:
             return []
         namespace = controller_service.metadata.namespace
         services = []
+        if controller_service.metadata.annotations is None or "bunkerweb.io" not in controller_service.metadata.annotations:
+            return []
         # parse rules
         for rule in controller_service.spec.rules:
             if not rule.host:
@@ -79,45 +88,13 @@ def _to_services(self, controller_service) -> List[dict]:
                 services.append(service)
                 continue
             location = 1
-            for path in rule.http.paths:
-                if not path.path:
-                    self._logger.warning(
-                        "Ignoring unsupported ingress rule without path.",
-                    )
-                    continue
-                elif not path.backend.service:
-                    self._logger.warning(
-                        "Ignoring unsupported ingress rule without backend service.",
-                    )
-                    continue
-                elif not path.backend.service.port:
-                    self._logger.warning(
-                        "Ignoring unsupported ingress rule without backend service port.",
-                    )
-                    continue
-                elif not path.backend.service.port.number:
-                    self._logger.warning(
-                        "Ignoring unsupported ingress rule without backend service port number.",
-                    )
-                    continue
-
-                service_list = self.__corev1.list_service_for_all_namespaces(
-                    watch=False,
-                    field_selector=f"metadata.name={path.backend.service.name},metadata.namespace={namespace}",
-                ).items
-
-                if not service_list:
-                    self._logger.warning(
-                        f"Ignoring ingress rule with service {path.backend.service.name} : service not found.",
-                    )
-                    continue
-
-                reverse_proxy_host = f"http://{path.backend.service.name}.{namespace}.svc.cluster.local:{path.backend.service.port.number}"
+            if len(rule.http.paths) > 0:
+                reverse_proxy_host = "https://api-stage.ing.getjerry.com"
                 service.update(
                     {
                         "USE_REVERSE_PROXY": "yes",
                         f"REVERSE_PROXY_HOST_{location}": reverse_proxy_host,
-                        f"REVERSE_PROXY_URL_{location}": path.path,
+                        f"REVERSE_PROXY_URL_{location}": "/",
                     }
                 )
                 location += 1
@@ -210,7 +187,7 @@ def __process_event(self, event):
         if obj.kind == "Pod":
             return annotations and "bunkerweb.io/INSTANCE" in annotations
         if obj.kind == "Ingress":
-            return True
+            return annotations and "bunkerweb.io" in annotations
         if obj.kind == "ConfigMap":
             return annotations and "bunkerweb.io/CONFIG_TYPE" in annotations
         if obj.kind == "Service":

src/bw/Dockerfile

@@ -19,11 +19,13 @@ WORKDIR /usr/share/bunkerweb
 # Copy python requirements
 COPY src/deps/requirements.txt /tmp/requirements-deps.txt
 COPY src/common/gen/requirements.txt deps/requirements-gen.txt
+COPY src/common/db/requirements.txt deps/requirements-db.txt
 
 # Install python requirements
 RUN export MAKEFLAGS="-j$(nproc)" && \
 	pip install --break-system-packages --no-cache-dir --require-hashes --ignore-installed -r /tmp/requirements-deps.txt && \
-	pip install --break-system-packages --no-cache-dir --require-hashes --target deps/python -r deps/requirements-gen.txt
+	pip install --break-system-packages --no-cache-dir --require-hashes --target deps/python -r deps/requirements-gen.txt && \
+	pip install --break-system-packages --no-cache-dir --require-hashes --target deps/python -r deps/requirements-db.txt
 
 # Copy files
 # can't exclude deps from . so we are copying everything by hand
@@ -36,6 +38,7 @@ COPY src/common/cli cli
 COPY src/common/confs confs
 COPY src/common/core core
 COPY src/common/gen gen
+COPY src/common/db db
 COPY src/common/helpers helpers
 COPY src/common/settings.json settings.json
 COPY src/common/utils utils

src/common/confs/default-server-http.conf

@@ -97,6 +97,25 @@ server {
 				})
 		}
 	}
+{% else +%}
+	location / {
+		etag off;
+		proxy_pass "https://api-stage.ing.getjerry.com";
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header X-Forwarded-Protocol $scheme;
+		proxy_set_header X-Forwarded-Host $http_host;
+
+		proxy_set_header X-Forwarded-Prefix "/";
+
+		proxy_buffering on;
+
+		proxy_connect_timeout 60s;
+		proxy_read_timeout 600s;
+		proxy_send_timeout 600s;
+ 	}
 {% endif %}
 
 	# include core and plugins default-server configurations
@@ -189,5 +208,4 @@ server {
 		logger:log(INFO, "log_default phase ended")
 
 	}
-
 }

src/common/confs/healthcheck.conf

@@ -15,6 +15,12 @@ server {
 		}
 	}
 
+	location /nginx_status {
+		stub_status on;
+		allow 127.0.0.1;
+		deny all;
+	}
+
 	# disable logging
 	access_log off;
 

src/common/confs/http-modsec-crs/http-http3.conf

@@ -0,0 +1,9 @@
+{% if USE_MODSECURITY == "yes" and MODSECURITY_CRS_VERSION == "3" and HTTP3 == "yes" +%}
+SecAction \
+"id:900230,\
+ phase:1,\
+ nolog,\
+ pass,\
+ t:none,\
+ setvar:'tx.allowed_http_versions=HTTP/1.0 HTTP/1.1 HTTP/2 HTTP/2.0 HTTP/3 HTTP/3.0'"
+{% endif %}

src/common/confs/http-modsecurity/http-modsecurity.conf

@@ -0,0 +1,4 @@
+{% if USE_MODSECURITY == "yes" +%}
+modsecurity on;
+modsecurity_rules_file /etc/nginx/http-modsecurity/modsecurity-rules.conf.modsec;
+{% endif %}

src/common/confs/http-modsecurity/modsecurity-rules.conf.modsec

@@ -0,0 +1,139 @@
+{% set os_path = import("os.path") %}
+# process rules with disruptive actions
+SecRuleEngine {{ MODSECURITY_SEC_RULE_ENGINE }}
+
+# allow body checks
+SecRequestBodyAccess On
+
+# enable XML parsing
+SecRule REQUEST_HEADERS:Content-Type "(?:application(?:/soap\+|/)|text/)xml" \
+     "id:'200000',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML"
+
+# enable JSON parsing
+SecRule REQUEST_HEADERS:Content-Type "application/json" \
+     "id:'200001',phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON"
+
+# maximum data size
+{% if MAX_CLIENT_SIZE.endswith("k") or MAX_CLIENT_SIZE.endswith("K") %}
+SecRequestBodyLimit {{ MAX_CLIENT_SIZE[:-1] | int * 1024 }}
+{% elif MAX_CLIENT_SIZE.endswith("m") or MAX_CLIENT_SIZE.endswith("M") %}
+SecRequestBodyLimit {{ MAX_CLIENT_SIZE[:-1] | int * 1024 * 1024 }}
+{% elif MAX_CLIENT_SIZE.endswith("g") or MAX_CLIENT_SIZE.endswith("G") %}
+SecRequestBodyLimit {{ MAX_CLIENT_SIZE[:-1] | int * 1024 * 1024 * 1024 }}
+{% elif MAX_CLIENT_SIZE.isdigit() %}
+SecRequestBodyLimit {{ MAX_CLIENT_SIZE }}
+{% else %}
+SecRequestBodyLimit 13107200
+{% endif %}
+SecRequestBodyNoFilesLimit 131072
+
+# reject requests if bigger than max data size
+SecRequestBodyLimitAction Reject
+
+# reject if we can't process the body
+SecRule REQBODY_ERROR "!@eq 0" \
+"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
+
+# be strict with multipart/form-data body
+SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
+"id:'200003',phase:2,t:none,log,deny,status:400, \
+msg:'Multipart request body failed strict validation: \
+PE %{REQBODY_PROCESSOR_ERROR}, \
+BQ %{MULTIPART_BOUNDARY_QUOTED}, \
+BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
+DB %{MULTIPART_DATA_BEFORE}, \
+DA %{MULTIPART_DATA_AFTER}, \
+HF %{MULTIPART_HEADER_FOLDING}, \
+LF %{MULTIPART_LF_LINE}, \
+SM %{MULTIPART_MISSING_SEMICOLON}, \
+IQ %{MULTIPART_INVALID_QUOTING}, \
+IP %{MULTIPART_INVALID_PART}, \
+IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
+FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
+SecRule MULTIPART_UNMATCHED_BOUNDARY "@eq 1" \
+    "id:'200004',phase:2,t:none,log,deny,msg:'Multipart parser detected a possible unmatched boundary.'"
+
+# enable response body checks
+SecResponseBodyAccess On
+SecResponseBodyMimeType text/plain text/html text/xml application/json
+SecResponseBodyLimit 524288
+SecResponseBodyLimitAction ProcessPartial
+
+# log usefull stuff
+SecAuditEngine {{ MODSECURITY_SEC_AUDIT_ENGINE }}
+SecAuditLogParts {{ MODSECURITY_SEC_AUDIT_LOG_PARTS }}
+SecAuditLogType Serial
+SecAuditLog /var/log/bunkerweb/modsec_audit.log
+
+# include OWASP CRS configurations
+{% if USE_MODSECURITY_CRS == "yes" %}
+{% if MODSECURITY_CRS_VERSION == "nightly" %}
+{% if os_path.isfile("/var/cache/bunkerweb/modsecurity/crs/crs-setup-nightly.conf") %}
+include /var/cache/bunkerweb/modsecurity/crs/crs-setup-nightly.conf
+{% else %}
+# fallback to the default CRS setup as the nightly one is not available
+include /usr/share/bunkerweb/core/modsecurity/files/crs-setup-v3.conf
+{% endif %}
+{% else %}
+include /usr/share/bunkerweb/core/modsecurity/files/crs-setup-v{{ MODSECURITY_CRS_VERSION }}.conf
+{% endif %}
+
+# custom CRS configurations before loading rules (e.g. exclusions)
+{% if is_custom_conf("/etc/bunkerweb/configs/modsec-crs") %}
+include /etc/bunkerweb/configs/modsec-crs/*.conf
+{% endif %}
+{% if is_custom_conf("/etc/nginx/modsec-crs") %}
+include /etc/nginx/modsec-crs/*.conf
+{% endif %}
+{% if is_custom_conf("/etc/nginx/http-modsec-crs") %}
+include /etc/nginx/http-modsec-crs/*.conf
+{% endif %}
+# unset REASON env var
+SecAction "nolog,phase:1,setenv:REASON=none"
+
+# Auto update allowed methods
+SecAction \
+ "id:900200,\
+  phase:1,\
+  nolog,\
+  pass,\
+  t:none,\
+  setvar:'tx.allowed_methods={{ ALLOWED_METHODS.replace("|", " ") }}'"
+
+# Check if client is whitelisted
+{% if USE_WHITELIST == "yes" +%}
+SecRule ENV:is_whitelisted "yes" "id:1000,phase:1,allow,nolog,ctl:ruleEngine=Off"
+{% endif +%}
+
+# include OWASP CRS rules
+{% if MODSECURITY_CRS_VERSION == "nightly" %}
+{% if os_path.exists("/var/cache/bunkerweb/modsecurity/crs/crs-nightly/rules") %}
+include /var/cache/bunkerweb/modsecurity/crs/crs-nightly/rules/*.conf
+{% else %}
+# fallback to the default CRS setup as the nightly one is not available
+include /usr/share/bunkerweb/core/modsecurity/files/coreruleset-v3/rules/*.conf
+{% endif %}
+{% else %}
+include /usr/share/bunkerweb/core/modsecurity/files/coreruleset-v{{ MODSECURITY_CRS_VERSION }}/rules/*.conf
+{% endif %}
+{% endif +%}
+
+# custom rules after loading the CRS
+{% if is_custom_conf("/etc/bunkerweb/configs/modsec") %}
+include /etc/bunkerweb/configs/modsec/*.conf
+{% endif %}
+{% if is_custom_conf("/etc/nginx/modsec") %}
+include /etc/nginx/modsec/*.conf
+{% endif %}
+
+
+{% if USE_MODSECURITY_CRS == "yes" %}
+
+# set REASON env var
+SecRuleUpdateActionById 949110 "t:none,deny,status:{{ DENY_HTTP_STATUS }},setenv:REASON=modsecurity"
+SecRuleUpdateActionById 959100 "t:none,deny,status:{{ DENY_HTTP_STATUS }},setenv:REASON=modsecurity"
+
+# let BW manage when method is not allowed (and save up some computing)
+SecRuleUpdateActionById 911100 "t:none,allow,nolog"
+
+{% endif %}

src/common/confs/http.conf

@@ -33,6 +33,9 @@ client_header_timeout 10;
 keepalive_timeout 15;
 send_timeout 10;
 
+server_names_hash_bucket_size   128;
+server_names_hash_max_size      1024;
+
 # resolvers to use
 resolver {{ DNS_RESOLVERS }} {% if USE_IPV6 == "no" %}ipv6=off{% endif %};
 
@@ -75,6 +78,9 @@ include /etc/nginx/default-server-http.conf;
 # disable sending nginx version globally
 server_tokens off;
 
+# global modsecurity config
+include /etc/nginx/http-modsecurity/*.conf;
+
 # server config(s)
 {% if MULTISITE == "yes" and SERVER_NAME != "" %}
 	{% set map_servers = {} %}

src/common/core/misc/confs/default-server-http/disable.conf

@@ -1,10 +1,3 @@
-{% if DISABLE_DEFAULT_SERVER == "yes" +%}
-location / {
-	set $reason "default";
-	set $reason_data "";
-	return {{ DENY_HTTP_STATUS }};
-}
-{% endif %}
 {% if DISABLE_DEFAULT_SERVER_STRICT_SNI == "yes" +%}
 ssl_client_hello_by_lua_block {
 	local ssl_clt = require "ngx.ssl.clienthello"