⚡️ Improve file serving in API server

ansible/roles/api-server/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
   - common
+  - nginx
   - nodejs

ansible/roles/docker/tasks/main.yml

@@ -2,13 +2,10 @@
   become: true
   block:
     - name: Add signing key
-      ansible.builtin.uri:
+      ansible.builtin.get_url:
         url: https://download.docker.com/linux/debian/gpg
         dest: /etc/apt/keyrings/docker.asc
         mode: '644'
-        status_code:
-          - 200 # OK
-          - 304 # Not Modified
 
     - name: Add repository
       ansible.builtin.apt_repository:

ansible/roles/nginx/files/nginx.conf

@@ -62,7 +62,7 @@ http {
 	# gzip_comp_level 6;
 	# gzip_buffers 16 8k;
 	# gzip_http_version 1.1;
-	# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+	gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
 
 	##
 	# Virtual Host Configs

package.json

@@ -19,8 +19,8 @@
 		"docs:start": "cd docs && (bundle exec jekyll serve --incremental || true) && cd ..",
 		"fmt": "dprint fmt",
 		"fmt:check": "dprint check",
-		"lint": "eslint --cache --max-warnings=0 packages/*/{src,test}/**/*.{cts,mts,ts}",
-		"lint:fix": "eslint --cache --max-warnings=0 --fix packages/*/{src,test}/**/*.{cts,mts,ts}",
+		"lint": "eslint --cache --max-warnings=0 \"packages/*/{src,test}/**/*.{cts,mts,ts}\"",
+		"lint:fix": "eslint --cache --max-warnings=0 --fix \"packages/*/{src,test}/**/*.{cts,mts,ts}\"",
 		"release": "ts-node scripts/release.ts",
 		"release:dry": "ts-node scripts/release.ts --dry-run",
 		"set-tsconfig-references": "ts-node scripts/set_tsconfig_references",

packages/web-api-server/package.json

@@ -16,19 +16,16 @@
     "start": "./bin/server.js"
   },
   "dependencies": {
-    "async-mutex": "^0.5.0",
     "chalk": "^5.4.1",
     "cors": "^2.8.5",
     "express": "^5.0.1",
     "express-slow-down": "^2.0.3",
     "rate-limiter-flexible": "^5.0.4",
-    "simple-git": "^3.27.0",
-    "tar-fs": "^3.0.6"
+    "simple-git": "^3.27.0"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",
-    "@types/express": "^5.0.0",
-    "@types/tar-fs": "^2.0.4"
+    "@types/express": "^5.0.0"
   },
   "publishConfig": {
     "access": "public"

packages/web-api-server/src/index.ts

@@ -12,6 +12,7 @@ import {
 	initGitRepos,
 	loadConfig,
 	loggerMiddleware as logger,
+	MemCache,
 	sendGitFile,
 	sendGitTarball,
 	userAgentEnforcer,
@@ -20,34 +21,30 @@ import {
 
 const { hookSecret, port, rootDir } = loadConfig()
 await assertRootDir(rootDir)
-const repos = await initGitRepos(rootDir)
+const gits = await initGitRepos(rootDir)
+const cache = new MemCache(gits.mcmeta)
 
 const versionRoute = express.Router({ mergeParams: true })
-	.use(getVersionValidator(repos.summary.git))
+	.use(getVersionValidator(cache))
 	.get('/block_states', cheapRateLimiter, async (req, res) => {
 		const { version } = req.params
-		const { git } = repos.summary
-		await sendGitFile(req, res, git, `${version}-summary`, 'blocks/data.json')
+		await sendGitFile(req, res, gits.mcmeta, `${version}-summary`, 'blocks/data.json')
 	})
 	.get('/commands', cheapRateLimiter, async (req, res) => {
 		const { version } = req.params
-		const { git } = repos.summary
-		await sendGitFile(req, res, git, `${version}-summary`, 'commands/data.json')
+		await sendGitFile(req, res, gits.mcmeta, `${version}-summary`, 'commands/data.json')
 	})
 	.get('/registries', cheapRateLimiter, async (req, res) => {
 		const { version } = req.params
-		const { git } = repos.summary
-		await sendGitFile(req, res, git, `${version}-summary`, 'registries/data.json')
+		await sendGitFile(req, res, gits.mcmeta, `${version}-summary`, 'registries/data.json')
 	})
 	.get('/vanilla-assets-tiny/tarball', expensiveRateLimiter, async (req, res) => {
 		const { version } = req.params
-		const { git, mutex, repoDir } = repos.assets
-		await sendGitTarball(req, res, git, mutex, repoDir, `${version}-assets-tiny`)
+		await sendGitTarball(req, res, gits.mcmeta, `${version}-assets-tiny`)
 	})
 	.get('/vanilla-data/tarball', expensiveRateLimiter, async (req, res) => {
 		const { version } = req.params
-		const { git, mutex, repoDir } = repos.data
-		await sendGitTarball(req, res, git, mutex, repoDir, `${version}-data`)
+		await sendGitTarball(req, res, gits.mcmeta, `${version}-data`)
 	})
 
 const DELAY_AFTER = 50
@@ -72,17 +69,14 @@ const app = express()
 		keyGenerator: (req) => req.ip!.replace(/:\d+[^:]*$/, ''),
 	}))
 	.get('/mcje/versions', cheapRateLimiter, async (req, res) => {
-		const { git } = repos.summary
-		await sendGitFile(req, res, git, 'summary', 'versions/data.json')
+		await sendGitFile(req, res, gits.mcmeta, 'summary', 'versions/data.json')
 	})
 	.use('/mcje/versions/:version', versionRoute)
 	.get('/vanilla-mcdoc/symbols', cheapRateLimiter, async (req, res) => {
-		const { git } = repos.mcdoc
-		await sendGitFile(req, res, git, `generated`, 'symbols.json')
+		await sendGitFile(req, res, gits.mcdoc, `generated`, 'symbols.json')
 	})
 	.get('/vanilla-mcdoc/tarball', expensiveRateLimiter, async (req, res) => {
-		const { git, mutex, repoDir } = repos.mcdoc
-		await sendGitTarball(req, res, git, mutex, repoDir, 'main', 'vanilla-mcdoc')
+		await sendGitTarball(req, res, gits.mcdoc, 'main', 'vanilla-mcdoc')
 	})
 	.post(
 		'/hooks/github',
@@ -98,25 +92,17 @@ const app = express()
 			}
 			res.status(202).send(JSON.stringify({ message: 'Accepted' }))
 
-			// if (req.headers['x-github-event'] !== 'push') {
-			// 	return
-			// }
+			if (req.headers['x-github-event'] !== 'push') {
+				return
+			}
 
 			const { repository: { name } } = JSON.parse(req.body.toString()) as {
 				repository: { name: string }
 			}
-			const reposToUpdate = name === 'mcmeta'
-				? [repos.assets, repos.data, repos.summary]
-				: name === 'vanilla-mcdoc'
-				? [repos.mcdoc]
-				: []
-			for (const { git, mutex, repoDir } of reposToUpdate) {
-				await mutex.runExclusive(async () => {
-					console.info(chalk.yellow(`Pulling into ${repoDir}...`))
-					await git.pull()
-					console.info(chalk.green(`Pulled into ${repoDir}`))
-				})
-			}
+			const git = gits[name === 'vanilla-mcdoc' ? 'mcdoc' : 'mcmeta']
+			console.info(chalk.yellow(`Updating ${name}...`))
+			await git.remote(['update', '--prune'])
+			console.info(chalk.green(`Updated ${name}`))
 		},
 	)
 	.get('/favicon.ico', cheapRateLimiter, (_req, res) => {