Merge remote-tracking branch 'upstream/master'

app.js

@@ -63,6 +63,7 @@ debug.enable(process.env.DEBUG || debugDefaultCategories);
 
 
 global.cacheStats = {};
+global.appEventStats = {};
 
 
 
@@ -118,6 +119,7 @@ debugLog(`Default cacheId '${global.cacheId}'`);
 global.btcNodeSemver = "0.0.0";
 
 
+const cleanupRouter = require('./routes/cleanupRouter.js');
 const baseActionsRouter = require('./routes/baseRouter.js');
 const internalApiActionsRouter = require('./routes/internalApiRouter.js');
 const apiActionsRouter = require('./routes/apiRouter.js');
@@ -255,6 +257,14 @@ if (rateLimitWindowMinutes == -1) {
 		standardHeaders: 'draft-7', // draft-6: `RateLimit-*` headers; draft-7: combined `RateLimit` header
 		legacyHeaders: false, // Disable the `X-RateLimit-*` headers.
 		skip: function (req, res) {
+			// tor traffic all comes in via tor proxy showing 127.0.0.1
+			// for now, until we identify it as a serious problem, let it pass
+			if (req.hostname.includes(".onion")) {
+				utils.trackAppEvent("torRequest");
+
+				return true;
+			}
+
 			if (req.originalUrl.includes("/snippet/")) {
 				return true;
 			}
@@ -266,7 +276,10 @@ if (rateLimitWindowMinutes == -1) {
 			return false;
 		},
 		handler: function (req, res, next) {
-			debugErrorLog(`Rate-limiting request: ip=${req.ip}, req=${req.originalUrl}`)
+			debugErrorLog(`Rate-limiting request: req=${JSON.stringify(utils.expressRequestToJson(req))}`);
+
+			utils.trackAppEvent("rateLimitedRequest");
+
 			res.status(429).json({
 				message: "Too many requests, please try again later.",
 			});
@@ -581,17 +594,30 @@ async function onRpcConnectionVerified(getnetworkinfo, getblockchaininfo) {
 
 
 	if (false) {
-		var zmq = require("zeromq");
-		var sock = zmq.socket("sub");
+		monitorNewTransactions().catch(err => console.error(err));
 
-		sock.connect("tcp://192.168.1.1:21331");
-		console.log("Worker connected to port 21331");
+		//sock.subscribe('rawtx');
+	}
+}
 
-		sock.on("message", function(topic, message) {
-			console.log(Buffer.from(topic).toString("ascii") + " - " + Buffer.from(message).toString("hex"));
-		});
+async function monitorNewTransactions() {
+	const zmq = require("zeromq");
+	const sock = new zmq.Subscriber();
 
-		//sock.subscribe('rawtx');
+	sock.connect("tcp://ubuntu:21331");
+	console.log("Worker connected to port 21331");
+
+	// Subscribe to all topics (use sock.subscribe("specific_topic") for specific topics)
+	sock.subscribe();
+
+	for await (const [topic, message] of sock) {
+		utils.trackAppEvent("newTransaction");
+
+		console.log(
+			topic.toString("ascii") +
+			" - " +
+			message.toString("hex")
+		);
 	}
 }
 
@@ -1025,6 +1051,8 @@ expressApp.continueStartup = function() {
 };
 
 expressApp.use(function(req, res, next) {
+	utils.trackAppEvent("request");
+
 	req.startTime = Date.now();
 
 	next();
@@ -1142,6 +1170,7 @@ expressApp.use(csrfProtection, (req, res, next) => {
 	next();
 });
 
+expressApp.use(config.baseUrl, cleanupRouter);
 expressApp.use(config.baseUrl, baseActionsRouter);
 expressApp.use(config.baseUrl + 'internal-api/', internalApiActionsRouter);
 expressApp.use(config.baseUrl + 'api/', apiActionsRouter);
@@ -1173,6 +1202,8 @@ expressApp.use(function(req, res, next) {
 
 /// catch 404 and forwarding to error handler
 expressApp.use(function(req, res, next) {
+	utils.trackAppEvent("error404");
+
 	var err = new Error(`Not Found: ${req ? req.url : 'unknown url'}`);
 	err.status = 404;
 
@@ -1192,6 +1223,8 @@ const sharedErrorHandler = (req, err) => {
 
 		if (crawler) {
 			attributes.crawler = crawler;
+
+			utils.trackAppEvent("crawlRequest", 1, {"crawler": crawler});
 		}
 
 		debugErrorLog(`404 NotFound: path=${path}, ip=${ip}, userAgent=${userAgent} (crawler=${(crawler != null)}${crawler ? crawler : ""})`);

app/resourceIntegrityHashes.js

@@ -12,11 +12,11 @@ module.exports =
     "sentry.min.js": "sha384-da/Bo2Ah6Uw3mlhl6VINMblg2SyGbSnULKrukse3P5D9PTJi4np9HoKvR19D7zOL",
     "site.js": "sha384-G8o2io5zIdQiiVN+4CAGGXPO4UvV8jGkSMlfddSPbqIUd3x5Rv01pNH6ec8QOATu",
     "bootstrap-icons.css": "sha384-rJFhkIguED0Z4GX6r6ReHpTCkwWtiPHZnQtWVP0DQWcKHzeJAlYb1m/xdYkeEk+f",
-    "dark-v1.min.css": "sha384-3ru8Vuc+jzkyRBZkRRdFb+O2gHR43bYmGOoqCL20MNGNTC2cZw6mD+L+mAW/bWvU",
-    "dark.min.css": "sha384-mOoFwflyWEIHcyYPxrf7P+JX+LM3XLjDp5nZRPExl67nMFb0zD9/dycqYRdSYj8G",
+    "dark-v1.min.css": "sha384-NJK2FF+pcgXOxhCACC4A35Noub6HxksU2B0JXRUWaCqpSksqX+6UWUG1wXugc/8W",
+    "dark.min.css": "sha384-+DbPLnXtdAT+C2GMbf14OD2QDEPgBDF47Vf/EzY2sdOrg8stc+u3yqvKYqnct5Ck",
     "dataTables.bootstrap4.min.css": "sha384-EkHEUZ6lErauT712zSr0DZ2uuCmi3DoQj6ecNdHQXpMpFNGAQ48WjfXCE5n20W+R",
     "highlight.min.css": "sha384-s4RLYRjGGbVqKOyMGGwfxUTMOO6D7r2eom7hWZQ6BjK2Df4ZyfzLXEkonSm0KLIQ",
-    "light.min.css": "sha384-v0ZyOfDZIvzUB1WV4KP6IcUjWcyv998Ghr1oFbdtwTD5jt72cgG4bNlRTLa1j3f8",
+    "light.min.css": "sha384-53K2dzYpqW7F28D8HeJ+m+FFa7gXjm+N7iNchAn0bqB3cyKHW1cpq2lNyoN+4nmf",
     "leaflet.css": "sha384-6wKUKNzA6h/S6gZ1lWQppeGaVXvK1AUAsEznGBghzlEu1fNcxJGYVRiroSHr+OwU",
     "leaflet.js": "sha384-RFZC58YeKApoNsIbBxf4z6JJXmh+geBSgkCQXFyh+4tiFSJmJBt+2FbjxW7Ar16M"
 };

app/utils.js

@@ -64,6 +64,9 @@ const crawlerBotUserAgentStrings = {
 	"python-requests": new RegExp("python-requests", "i"),
 	"openai": new RegExp("OAI-SearchBot", "i"),
 	"unidentifiedCrawler": new RegExp("Test Certificate Info", "i"),
+	"amazon": new RegExp("amazonbot", "i"),
+	"bytedance": new RegExp("bytespider", "i"),
+	"scrapy": new RegExp("Scrapy", "i"),
 };
 
 const ipMemoryCache = {};
@@ -977,7 +980,7 @@ function logError(errorId, err, optionalUserData = {}, logStacktrace=true) {
 		};
 	}
 
-	if (optionalUserData && err.message) {
+	if (optionalUserData && err && err.message) {
 		optionalUserData.errorMsg = err.message;
 	}
 
@@ -1371,6 +1374,26 @@ function bip32Addresses(extPubkey, addressType, account, limit=10, offset=0) {
 	return addresses;
 }
 
+function expressRequestToJson(req) {
+	return {
+		method: req.method,
+		url: req.url,
+		headers: req.headers,
+		query: req.query,
+		params: req.params,
+		body: req.body,
+		cookies: req.cookies,
+		signedCookies: req.signedCookies,
+		ip: req.ip,
+		ips: req.ips,
+		protocol: req.protocol,
+		secure: req.secure,
+		originalUrl: req.originalUrl,
+		hostname: req.hostname,
+		baseUrl: req.baseUrl,
+	};
+}
+
 function difficultyAdjustmentEstimates(eraStartBlockHeader, currentBlockHeader) {
 	let difficultyPeriod = parseInt(Math.floor(currentBlockHeader.height / coinConfig.difficultyAdjustmentBlockCount));
 	let blocksUntilDifficultyAdjustment = ((difficultyPeriod + 1) * coinConfig.difficultyAdjustmentBlockCount) - currentBlockHeader.height;
@@ -1607,6 +1630,31 @@ const perfLogNewItem = (tags) => {
 	};
 };
 
+function trackAppEvent(name, count=1, params=null) {
+	if (!global.appEventStats[name]) {
+		global.appEventStats[name] = {count:0};
+	}
+
+	global.appEventStats[name].count += count;
+	global.appEventStats[name].last = new Date();
+
+	if (params != null) {
+		if (global.appEventStats[name].params == null) {
+			global.appEventStats[name].params = {};
+		}
+
+		let props = objectProperties(params);
+
+		props.forEach(prop => {
+			if (global.appEventStats[name].params[`${prop}[${params[prop]}]`] == null) {
+				global.appEventStats[name].params[`${prop}[${params[prop]}]`] = {count: 0};
+			}
+
+			global.appEventStats[name].params[`${prop}[${params[prop]}]`].count += count;
+		});
+	}
+}
+
 module.exports = {
 	reflectPromise: reflectPromise,
 	redirectToConnectPageIfNeeded: redirectToConnectPageIfNeeded,
@@ -1669,5 +1717,7 @@ module.exports = {
 	awaitPromises: awaitPromises,
 	perfLogNewItem: perfLogNewItem,
 	perfLog: perfLog,
-	fileCache: fileCache
+	fileCache: fileCache,
+	expressRequestToJson: expressRequestToJson,
+	trackAppEvent: trackAppEvent
 };

docs/Server-Setup.md

@@ -8,10 +8,11 @@ Update and install packages
 
 Install NVM from https://github.com/nvm-sh/nvm
 
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
     nvm ls-remote
 
     # install latest node from output of ls-remote above, e.g.:
-    nvm install 15.13.0
+    nvm install 22.11.0
 
     npm install -g pm2
 

package.json

@@ -13,14 +13,13 @@
 		"css-light-debug": "sass --style expanded ./public/scss/light.scss ./public/style/light.css",
 		"css-dark-debug": "sass --style expanded ./public/scss/dark.scss ./public/style/dark.css",
 		"css-dark-v1-debug": "sass --style expanded ./public/scss/dark-v1.scss ./public/style/dark-v1.css",
-		"css-debug": "npm-run-all css-light-debug css-dark-debug css-dark-v1-debug",
+		"css-debug": "npm run css-light-debug && npm run css-dark-debug && npm run css-dark-v1-debug",
 		"css-light": "sass --style compressed ./public/scss/light.scss ./public/style/light.min.css",
 		"css-dark": "sass --style compressed ./public/scss/dark.scss ./public/style/dark.min.css",
 		"css-dark-v1": "sass --style compressed ./public/scss/dark-v1.scss ./public/style/dark-v1.min.css",
-		"css": "npm-run-all css-light css-dark css-dark-v1 integrity"
+		"css": "npm run css-light && npm run css-dark && npm run css-dark-v1 && npm run integrity"
 	},
-	"keywords":
-	[
+	"keywords": [
 		"groestlcoin",
 		"grs",
 		"blockchain"
@@ -46,7 +45,7 @@
 		"compression": "^1.7.4",
 		"cookie-parser": "^1.4.6",
 		"crypto-js": "^4.2.0",
-		"csurf": "^1.11.0",
+		"csurf": "^1.2.2",
 		"debug": "^4.3.4",
 		"decimal.js": "^10.4.3",
 		"dotenv": "^13.0.1",
@@ -73,14 +72,13 @@
 		"serve-favicon": "^2.5.0",
 		"simple-git": "^3.18.0",
 		"tiny-secp256k1": "^2.2.1",
-		"zeromq": "^5.3.1"
+		"zeromq": "^6.1.2"
 	},
 	"optionalDependencies": {
 		"event-loop-stats": "^1.4.1"
 	},
 	"devDependencies": {
 		"eslint": "^8.39.0",
-		"npm-run-all": "^4.1.5",
-		"sass": "^1.62.1"
+		"sass": "^1.81.0"
 	}
 }

routes/adminRouter.js

@@ -37,6 +37,7 @@ router.get("/dashboard", function(req, res, next) {
 	res.locals.electrumStats = global.electrumStats;
 	res.locals.cacheStats = global.cacheStats;
 	res.locals.errorStats = global.errorStats;
+	res.locals.appEventStats = global.appEventStats;
 
 	res.locals.cacheSizes = {
 		misc: {

routes/baseRouter.js

@@ -1113,7 +1113,7 @@ router.get("/block-height/:blockHeight", asyncHandler(async (req, res, next) =>
 		next();
 
 	} catch (err) {
-		res.locals.userMessageMarkdown = `Failed loading block: height=**${blockHeight}**`;
+		res.locals.userMessageMarkdown = `Failed loading block: height=**${req.params.blockHeight}**`;
 
 		res.locals.pageErrors.push(utils.logError("389wer07eghdd", err));
 
@@ -1360,13 +1360,6 @@ router.get("/block-analysis", function(req, res, next) {
 	next();
 });
 
-/*
-router.get("/tx/tx/:transactionId", asyncHandler(async (req, res, next) => {
-	res.redirect(301, `${config.baseUrl}tx/${req.params.transactionId}`);
-
-	return;
-}));*/
-
 router.get("/tx/:transactionId@:blockHeight", asyncHandler(async (req, res, next) => {
 	req.query.blockHeight = req.params.blockHeight;
 	req.url = "/tx/" + req.params.transactionId;