Skip to content

Commit 494f9f1

Browse files
FalkWolskyFalkWolsky
authored
Merge pull request #293 from lowcoder-org/unify_max_request_size
new: unify usage of max request size across all components
2 parents 3239471 + c5a6854 commit 494f9f1

File tree

14 files changed

+206
-28
lines changed

14 files changed

+206
-28
lines changed

‎deploy/docker/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ RUN --mount=type=cache,target=/root/.m2 mvn -f pom.xml clean package -DskipTests
2020
RUN mkdir -p /lowcoder/api-service/plugins /lowcoder/api-service/config /lowcoder/api-service/logs
2121

2222
# Define lowcoder main jar and plugin jars
23-
ARG JAR_FILE=/lowcoder-server/lowcoder-server/target/lowcoder-server-1.0-SNAPSHOT.jar
23+
ARG JAR_FILE=/lowcoder-server/lowcoder-server/target/lowcoder-server-*.jar
2424
ARG PLUGIN_JARS=/lowcoder-server/lowcoder-plugins/*/target/*.jar
2525

2626
# Copy Java runtime for running server

‎deploy/docker/README.md

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ Image can be configured by setting environment variables.
3636
| `ENCRYPTION_PASSWORD` | Encryption password | `lowcoder.org` |
3737
| `ENCRYPTION_SALT` | Salt used for encrypting password | `lowcoder.org` |
3838
| `CORS_ALLOWED_DOMAINS` | CORS allowed domains | `*` |
39+
| `LOWCODER_MAX_REQUEST_SIZE` | Lowcoder max request size | `20m` |
3940
| `LOWCODER_API_SERVICE_URL` | Lowcoder API service URL | `http://localhost:8080` |
4041
| `LOWCODER_NODE_SERVICE_URL` | Lowcoder Node service (js executor) URL | `http://localhost:6060` |
4142
| `DEFAULT_ORGS_PER_USER` | Default maximum organizations per user | `100` |
@@ -121,6 +122,7 @@ Image can be configured by setting environment variables.
121122
| --------------------------------| --------------------------------------------------------------------| ------------------------------------------------------- |
122123
| `PUID` | ID of user running services. It will own all created logs and data. | `9001` |
123124
| `PGID` | ID of group of the user running services. | `9001` |
125+
| `LOWCODER_MAX_REQUEST_SIZE` | Lowcoder max request size | `20m` |
124126
| `LOWCODER_API_SERVICE_URL` | Lowcoder API service URL | `http://localhost:8080` |
125127
| `LOWCODER_NODE_SERVICE_URL` | Lowcoder Node service (js executor) URL | `http://localhost:6060` |
126128

‎deploy/docker/docker-compose-multi.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ services:
7575
environment:
7676
PUID: "9001"
7777
PGID: "9001"
78+
LOWCODER_MAX_REQUEST_SIZE: 20m
7879
LOWCODER_API_SERVICE_URL: "http://lowcoder-api-service:8080"
7980
LOWCODER_NODE_SERVICE_URL: "http://lowcoder-node-service:6060"
8081
restart: unless-stopped

‎deploy/docker/docker-compose.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ services:
3636
# api and node service parameters
3737
LOWCODER_API_SERVICE_URL: "http://localhost:8080"
3838
LOWCODER_NODE_SERVICE_URL: "http://localhost:6060"
39+
# frontend parameters
40+
LOWCODER_MAX_REQUEST_SIZE: 20m
3941
volumes:
4042
- ./lowcoder-stacks:/lowcoder-stacks
4143
restart: unless-stopped

‎deploy/docker/frontend/01-update-nginx-conf.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ else
1818
ln -s /etc/nginx/nginx-http.conf /etc/nginx/nginx.conf
1919
fi;
2020

21+
sed -i "s@__LOWCODER_MAX_REQUEST_SIZE__@${LOWCODER_MAX_REQUEST_SIZE:=20m}@" /etc/nginx/nginx.conf
2122
sed -i "s@__LOWCODER_API_SERVICE_URL__@${LOWCODER_API_SERVICE_URL:=http://localhost:8080}@" /etc/nginx/nginx.conf
2223
sed -i "s@__LOWCODER_NODE_SERVICE_URL__@${LOWCODER_NODE_SERVICE_URL:=http://localhost:6060}@" /etc/nginx/nginx.conf
2324

2425
echo "nginx config updated with:"
25-
echo " Lowcoder api service URL: ${LOWCODER_API_SERVICE_URL}"
26-
echo " Lowcoder node service URL: ${LOWCODER_NODE_SERVICE_URL}"
26+
echo " Lowcoder max upload size: ${LOWCODER_MAX_REQUEST_SIZE:=20m}"
27+
echo " Lowcoder api service URL: ${LOWCODER_API_SERVICE_URL:=http://localhost:8080}"
28+
echo " Lowcoder node service URL: ${LOWCODER_NODE_SERVICE_URL:=http://localhost:6060}"

‎deploy/docker/frontend/nginx-http.conf

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ http {
1010

1111
include /etc/nginx/mime.types;
1212
default_type application/octet-stream;
13+
client_max_body_size __LOWCODER_MAX_REQUEST_SIZE__;
1314

1415
log_format main '"$time_local" client=$remote_addr '
1516
'method=$request_method request="$request" '

‎deploy/docker/frontend/nginx-https.conf

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ http {
1010

1111
include /etc/nginx/mime.types;
1212
default_type application/octet-stream;
13+
client_max_body_size __LOWCODER_MAX_REQUEST_SIZE__;
1314

1415
log_format main '"$time_local" client=$remote_addr '
1516
'method=$request_method request="$request" '

‎server/api-service/lowcoder-sdk/src/main/java/org/lowcoder/sdk/config/CommonConfig.java

+36-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import java.util.HashSet;
77
import java.util.List;
88
import java.util.Set;
9+
import java.util.regex.Matcher;
10+
import java.util.regex.Pattern;
911

1012
import org.apache.commons.collections4.CollectionUtils;
1113
import org.apache.commons.collections4.ListUtils;
@@ -33,8 +35,9 @@ public class CommonConfig {
3335
private String version;
3436
private boolean blockHoundEnable;
3537
private String cookieName;
36-
private int maxQueryRequestSizeInMb = 10;
37-
private int maxQueryResponseSizeInMb = 10;
38+
private String maxUploadSize = "20MB";
39+
private String maxQueryRequestSize = "20MB";
40+
private String maxQueryResponseSize = "20MB";
3841
private Query query = new Query();
3942
private Cookie cookie = new Cookie();
4043
private JsExecutor jsExecutor = new JsExecutor();
@@ -48,6 +51,37 @@ public boolean isEnterpriseMode() {
4851
return workspace.getMode() == WorkspaceMode.ENTERPRISE;
4952
}
5053

54+
private static final Pattern HUMAN_DATA_SIZE_PATTERN = Pattern.compile("^([ \t]*)(?<number>\\d+(\\.\\d+)?)([ \t]*)(?<unit>[kKmMgGtT]{1})?([bB \t]*)$");
55+
public static String normalizeDataUnits(String dataWithUnits)
56+
{
57+
String normalized = dataWithUnits;
58+
Matcher m = HUMAN_DATA_SIZE_PATTERN.matcher(dataWithUnits);
59+
if (m.matches())
60+
{
61+
normalized = m.group("number");
62+
if (StringUtils.isNotBlank(m.group("unit")))
63+
{
64+
normalized += m.group("unit").toUpperCase() + "B";
65+
}
66+
}
67+
return normalized;
68+
}
69+
70+
public void setMaxUploadSize(String size)
71+
{
72+
this.maxUploadSize = normalizeDataUnits(size);
73+
}
74+
75+
public void setMaxQueryRequestSize(String size)
76+
{
77+
this.maxQueryRequestSize = normalizeDataUnits(size);
78+
}
79+
80+
public void setMaxQueryResponseSize(String size)
81+
{
82+
this.maxQueryResponseSize = normalizeDataUnits(size);
83+
}
84+
5185
@Data
5286
public static class Domain {
5387
private String defaultValue;

‎server/api-service/lowcoder-server/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,8 @@
153153
<artifactId>json-path</artifactId>
154154
</dependency>
155155
<dependency>
156-
<groupId>javax.servlet</groupId>
157-
<artifactId>javax.servlet-api</artifactId>
156+
<groupId>jakarta.servlet</groupId>
157+
<artifactId>jakarta.servlet-api</artifactId>
158158
</dependency>
159159

160160
<dependency>

‎server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/configuration/ApplicationConfiguration.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package org.lowcoder.api.framework.configuration;
2+
3+
import org.lowcoder.sdk.config.CommonConfig;
4+
import org.springframework.beans.factory.annotation.Autowired;
5+
import org.springframework.boot.web.servlet.MultipartConfigFactory;
6+
import org.springframework.context.annotation.Bean;
7+
import org.springframework.context.annotation.Configuration;
8+
import org.springframework.util.unit.DataSize;
9+
10+
import jakarta.servlet.MultipartConfigElement;
11+
12+
@Configuration
13+
public class ApplicationConfiguration
14+
{
15+
@Autowired
16+
private CommonConfig common;
17+
18+
@Bean
19+
public MultipartConfigElement multipartConfigElement()
20+
{
21+
MultipartConfigFactory factory = new MultipartConfigFactory();
22+
factory.setMaxRequestSize(DataSize.parse(common.getMaxUploadSize()));
23+
factory.setMaxFileSize(DataSize.parse(common.getMaxUploadSize()));
24+
return factory.createMultipartConfig();
25+
}
26+
}

‎server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/QueryExecuteHttpBodySizeFilter.java

+19-14
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
package org.lowcoder.api.framework.filter;
22

3-
import jakarta.annotation.PostConstruct;
4-
import lombok.extern.slf4j.Slf4j;
5-
import org.apache.commons.io.FileUtils;
3+
import static org.lowcoder.api.framework.filter.FilterOrder.QUERY_EXECUTE_HTTP_BODY_SIZE;
4+
import static org.lowcoder.sdk.exception.BizError.EXCEED_QUERY_REQUEST_SIZE;
5+
import static org.lowcoder.sdk.exception.BizError.EXCEED_QUERY_RESPONSE_SIZE;
6+
7+
import java.util.concurrent.atomic.AtomicLong;
8+
9+
import javax.annotation.Nonnull;
10+
611
import org.lowcoder.infra.constant.NewUrl;
712
import org.lowcoder.infra.constant.Url;
813
import org.lowcoder.sdk.config.CommonConfig;
@@ -19,19 +24,16 @@
1924
import org.springframework.http.server.reactive.ServerHttpResponse;
2025
import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
2126
import org.springframework.stereotype.Component;
27+
import org.springframework.util.unit.DataSize;
2228
import org.springframework.web.server.ServerWebExchange;
2329
import org.springframework.web.server.WebFilter;
2430
import org.springframework.web.server.WebFilterChain;
31+
32+
import jakarta.annotation.PostConstruct;
33+
import lombok.extern.slf4j.Slf4j;
2534
import reactor.core.publisher.Flux;
2635
import reactor.core.publisher.Mono;
2736

28-
import javax.annotation.Nonnull;
29-
import java.util.concurrent.atomic.AtomicLong;
30-
31-
import static org.lowcoder.api.framework.filter.FilterOrder.QUERY_EXECUTE_HTTP_BODY_SIZE;
32-
import static org.lowcoder.sdk.exception.BizError.EXCEED_QUERY_REQUEST_SIZE;
33-
import static org.lowcoder.sdk.exception.BizError.EXCEED_QUERY_RESPONSE_SIZE;
34-
3537
/**
3638
* check query request and response size
3739
*/
@@ -60,11 +62,13 @@ public Mono<Void> filter(@Nonnull ServerWebExchange exchange, @Nonnull WebFilter
6062
// check query api
6163
if (path.startsWith(NewUrl.QUERY_URL) || path.startsWith(Url.QUERY_URL)) {
6264

63-
long maxRequestSize = configInstance.ofLong("maxRequestSize",
64-
commonConfig.getMaxQueryRequestSizeInMb() * FileUtils.ONE_MB);
65-
long maxResponseSize = configInstance.ofLong("maxResponseSize",
66-
commonConfig.getMaxQueryResponseSizeInMb() * FileUtils.ONE_MB);
65+
String maxSize = configInstance.ofString("maxRequestSize", commonConfig.getMaxQueryRequestSize());
66+
long maxRequestSize = DataSize.parse(maxSize).toBytes();
67+
maxSize = configInstance.ofString("maxResponseSize", commonConfig.getMaxQueryResponseSize());
68+
long maxResponseSize = DataSize.parse(maxSize).toBytes();
6769

70+
log.info("Setting up maximum query request size to: {} bytes", maxRequestSize);
71+
log.info("Setting up maximum query response size to: {} bytes", maxResponseSize);
6872
ServerWebExchange newServerWebExchange = exchange.mutate()
6973
.request(new CustomServerHttpRequestDecorator(exchange.getRequest(), maxRequestSize))
7074
.response(new CustomServerHttpResponseDecorator(exchange.getResponse(), maxResponseSize))
@@ -126,4 +130,5 @@ public Mono<Void> writeWith(@Nonnull Publisher<? extends DataBuffer> body) {
126130
}));
127131
}
128132
}
133+
129134
}

‎server/api-service/lowcoder-server/src/main/resources/selfhost/ce/application.yml

+3-2
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ common:
4141
block-hound-enable: false
4242
js-executor:
4343
host: ${LOWCODER_NODE_SERVICE_URL:http://127.0.0.1:6060}
44-
max-query-request-size-in-mb: 20
45-
max-query-response-size-in-mb: 20
44+
max-query-request-size: ${LOWCODER_MAX_REQUEST_SIZE:20m}
45+
max-query-response-size: ${LOWCODER_MAX_REQUEST_SIZE:20m}
46+
max-upload-size: ${LOWCODER_MAX_REQUEST_SIZE:20m}
4647

4748
material:
4849
mongodb-grid-fs:

‎server/api-service/lowcoder-server/src/test/java/org/lowcoder/api/framework/configuration/CommonConfigTest.java

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package org.lowcoder.api.framework.configuration;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
5+
import org.junit.jupiter.api.Test;
6+
import org.lowcoder.sdk.config.CommonConfig;
7+
8+
public class CommonConfigTest
9+
{
10+
11+
/**
12+
* Test parsing with nginx notations (k, m, g)
13+
**/
14+
15+
@Test
16+
public void normalizeDataUnit_worksWithNginxBytes()
17+
{
18+
String nginx = "123456";
19+
assertEquals("123456", CommonConfig.normalizeDataUnits(nginx));
20+
}
21+
22+
@Test
23+
public void normalizeDataUnit_worksWithNginxKiloBytes()
24+
{
25+
String nginx = "16k";
26+
assertEquals("16KB", CommonConfig.normalizeDataUnits(nginx));
27+
}
28+
29+
@Test
30+
public void normalizeDataUnit_worksWithNginxMegaBytes()
31+
{
32+
String nginx = "20m";
33+
assertEquals("20MB", CommonConfig.normalizeDataUnits(nginx));
34+
}
35+
36+
@Test
37+
public void normalizeDataUnit_worksWithNginxGigaBytes()
38+
{
39+
String nginx = "20g";
40+
assertEquals("20GB", CommonConfig.normalizeDataUnits(nginx));
41+
}
42+
43+
/**
44+
* Test parsing with spring notations (B, KB, MB, GB)
45+
**/
46+
47+
@Test
48+
public void normalizeDataUnit_worksWithSpringBytes()
49+
{
50+
String nginx = "123456B";
51+
assertEquals("123456", CommonConfig.normalizeDataUnits(nginx));
52+
}
53+
54+
@Test
55+
public void normalizeDataUnit_worksWithSpringKiloBytes()
56+
{
57+
String nginx = "16KB";
58+
assertEquals("16KB", CommonConfig.normalizeDataUnits(nginx));
59+
}
60+
61+
@Test
62+
public void normalizeDataUnit_worksWithSpringMegaBytes()
63+
{
64+
String nginx = "20MB";
65+
assertEquals("20MB", CommonConfig.normalizeDataUnits(nginx));
66+
}
67+
68+
@Test
69+
public void normalizeDataUnit_worksWithSpringGigaBytes()
70+
{
71+
String nginx = "20GB";
72+
assertEquals("20GB", CommonConfig.normalizeDataUnits(nginx));
73+
}
74+
75+
/**
76+
* Test parsing with mixed case
77+
**/
78+
79+
@Test
80+
public void normalizeDataUnit_worksWithMixedCase()
81+
{
82+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10MB"));
83+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10mb"));
84+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10mB"));
85+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10m"));
86+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10M"));
87+
}
88+
89+
/**
90+
* Test parsing with whitespaces
91+
**/
92+
93+
@Test
94+
public void normalizeDataUnit_worksWithWhitespaces()
95+
{
96+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10 MB"));
97+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10M "));
98+
assertEquals("10MB", CommonConfig.normalizeDataUnits("10\tmb"));
99+
assertEquals("10MB", CommonConfig.normalizeDataUnits(" 10 MB "));
100+
assertEquals("10MB", CommonConfig.normalizeDataUnits("\t10\tMB\t"));
101+
assertEquals("10MB", CommonConfig.normalizeDataUnits(" \t 10 \tMB \t"));
102+
}
103+
}

‎server/api-service/pom.xml

+5-5
Original file line numberDiff line numberDiff line change
@@ -288,11 +288,11 @@
288288
<version>1.0.6.RELEASE</version>
289289
</dependency>
290290

291-
<dependency>
292-
<groupId>javax.servlet</groupId>
293-
<artifactId>javax.servlet-api</artifactId>
294-
<version>4.0.1</version>
295-
</dependency>
291+
<dependency>
292+
<groupId>jakarta.servlet</groupId>
293+
<artifactId>jakarta.servlet-api</artifactId>
294+
<version>6.0.0</version>
295+
</dependency>
296296

297297
<dependency>
298298
<groupId>io.projectreactor</groupId>

0 commit comments

Comments
 (0)