From 3e3e2032b291914680f5c7c23c586cb81d24add7 Mon Sep 17 00:00:00 2001 From: renxia Date: Wed, 8 May 2024 07:20:11 +0000 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E6=95=88=E7=8E=87=EF=BC=8C=E9=87=8D=E6=9E=84=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=20rule=20=E5=8A=A0=E8=BD=BD=E4=B8=8E=E5=A4=84?= =?UTF-8?q?=E7=90=86=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 - src/lib/getConfig.ts | 6 ++-- src/lib/ruleHandler.ts | 11 +++---- src/lib/{rulesManage.ts => rulesManager.ts} | 11 ++----- src/lib/update.ts | 7 +++-- src/lib/w2RulesManage.ts | 32 +++++++++++++++++---- src/rulesServer.ts | 5 ++-- src/server.ts | 8 +++--- src/tunnelRulesServer.ts | 18 +++++++----- typings/index.d.ts | 13 +++++---- 10 files changed, 69 insertions(+), 43 deletions(-) rename src/lib/{rulesManage.ts => rulesManager.ts} (95%) diff --git a/package.json b/package.json index 8bb4313..8893cfe 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "cookie", "env" ], - "packageManager": "pnpm@8.0.0", "engines": { "node": ">=16" }, diff --git a/src/lib/getConfig.ts b/src/lib/getConfig.ts index d979ce8..560d11e 100644 --- a/src/lib/getConfig.ts +++ b/src/lib/getConfig.ts @@ -11,7 +11,7 @@ import { basename, resolve } from 'node:path'; import { homedir } from 'node:os'; import { assign, color } from '@lzwme/fe-utils'; import { logger } from './helper'; -import { rulesManage } from './rulesManage'; +import { rulesManager } from './rulesManager'; import { Watcher } from './watch'; import { loadW2Rules } from './w2RulesManage'; @@ -93,12 +93,12 @@ export function getConfig(useCache = true) { } }); - const allRules = rulesManage.loadRules(config.ruleDirs, true); + const allRules = rulesManager.loadRules(config.ruleDirs, true); config.rules.forEach(d => { if (Array.isArray(d)) d.forEach(d => d.ruleId && allRules.set(d.ruleId, d)); else if (d?.ruleId) allRules.set(d.ruleId, d); }); - rulesManage.classifyRules([...allRules.values()], config, true); + rulesManager.classifyRules([...allRules.values()], config, true); loadW2Rules(config.whistleRules, 'clear'); diff --git a/src/lib/ruleHandler.ts b/src/lib/ruleHandler.ts index f1d48aa..bbd1dd8 100644 --- a/src/lib/ruleHandler.ts +++ b/src/lib/ruleHandler.ts @@ -2,7 +2,7 @@ * @Author: renxia * @Date: 2024-01-10 16:58:26 * @LastEditors: renxia - * @LastEditTime: 2024-03-05 15:33:27 + * @LastEditTime: 2024-05-08 15:19:51 * @Description: 基于 whistle 的 cookie 自动抓取插件 */ @@ -23,7 +23,7 @@ type RuleHandlerOptions = { resBody?: Buffer; }; -function ruleMatcher({ rule, req }: RuleHandlerOptions) { +export function ruleMatcher({ rule, req }: RuleHandlerOptions) { let errmsg = ''; // method @@ -87,7 +87,7 @@ export async function ruleHandler({ rule, req, res, reqBody, resBody }: RuleHand } } - const params: RuleHandlerParams = { req, reqBody: req._reqBody, resBody: res._resBody, headers, url, cookieObj, allCacheData: [], X }; + const params: RuleHandlerParams = { req, reqBody: req._reqBody, resBody: res._resBody, headers, url, cookieObj, cacheData: [], X }; if (resHeaders) params.resHeaders = resHeaders; if (rule.getCacheUid) { @@ -115,12 +115,13 @@ export async function ruleHandler({ rule, req, res, reqBody, resBody }: RuleHand cacheData[uid] = { update: now, data: { uid, headers: req.originalReq.headers, data: uidData } }; - params.allCacheData = []; + params.cacheData = []; const cacheDuration = 1000 * (Number(rule.cacheDuration || config.cacheDuration) || (rule.updateEnvValue ? 12 : 24 * 10) * 3600); for (const [key, value] of Object.entries(cacheData)) { if (cacheDuration && now - value.update > cacheDuration) delete cacheData[key]; - else params.allCacheData.push(value.data); + else params.cacheData.push(value.data); } + params.allCacheData = params.cacheData; storage.setItem(rule.ruleId, cacheData); } else { diff --git a/src/lib/rulesManage.ts b/src/lib/rulesManager.ts similarity index 95% rename from src/lib/rulesManage.ts rename to src/lib/rulesManager.ts index c50eb0d..a6e27fb 100644 --- a/src/lib/rulesManage.ts +++ b/src/lib/rulesManager.ts @@ -26,18 +26,13 @@ function ruleFormat(rule: RuleItem) { } if (!rule.on || !RuleOnList.includes(rule.on)) { - const ruleOn = rule.getCacheUid ? 'req-header' : 'res-body'; + const ruleOn = rule.getCacheUid ? 'res-body' : 'req-header'; if (rule.on) logger.warn(`[${rule.on}] 参数错误，自动设置为[${ruleOn}]！只能取值为: ${RuleOnList.join(', ')}`, rule.ruleId); rule.on = ruleOn; } if (!rule.desc) rule.desc = `${rule.ruleId}_${rule.url || rule.method}`; - if (!rule.mitm && typeof rule.url === 'string') { - const r = /^https:\/\/([a-z.*]+)/.exec(rule.url); - if (r) rule.mitm = [r[1]]; - } - if (rule.mitm) { if (!Array.isArray(rule.mitm)) rule.mitm = [rule.mitm]; rule.mitm = rule.mitm.filter(d => d && (typeof d === 'string' || d instanceof RegExp)); @@ -151,7 +146,7 @@ function removeRule(ruleId?: string[], filepath?: string[]) { const ruleSet = new Set(ruleId ? ruleId : []); const fileSet = new Set(filepath ? filepath : []); - for (const [type, item] of Object.entries(rulesManage.rules)) { + for (const [type, item] of Object.entries(rulesManager.rules)) { for (const [ruleId, rule] of item) { if (ruleSet.has(ruleId) || fileSet.has(rule._source)) { item.delete(ruleId); @@ -174,7 +169,7 @@ const onRuleFileChange: WatcherOnChange = (type, filepath) => { } }; -export const rulesManage = { +export const rulesManager = { rules: RulesCache, ruleFormat, classifyRules, diff --git a/src/lib/update.ts b/src/lib/update.ts index 25a7407..4542334 100644 --- a/src/lib/update.ts +++ b/src/lib/update.ts @@ -2,7 +2,7 @@ * @Author: renxia * @Date: 2024-01-11 13:38:34 * @LastEditors: renxia - * @LastEditTime: 2024-04-22 09:22:04 + * @LastEditTime: 2024-05-07 09:34:05 * @Description: */ import fs from 'node:fs'; @@ -22,7 +22,7 @@ export async function updateToQlEnvConfig(envConfig: EnvConfig, updateEnvValue?: if (!(await ql.login())) return; if (Date.now() - updateCache.updateTime > 1000 * 60 * 60 * 1) updateCache.qlEnvList = []; - let { name, value, desc, sep } = envConfig; + let { name, value, desc, sep = '\n' } = envConfig; let item = updateCache.qlEnvList.find(d => d.name === name); if (!item) { updateCache.qlEnvList = await ql.getEnvList(); @@ -62,7 +62,8 @@ export async function updateToQlEnvConfig(envConfig: EnvConfig, updateEnvValue?: } const isSuccess = r.code === 200; - logger.info(`${item ? green('更新') : magenta('新增')}QL环境变量[${name}]`, isSuccess ? '成功' : r); + const count = params.value.includes(sep) ? params.value.trim().split(sep).length : 1; + logger.info(`${item ? green('更新') : magenta('新增')}QL环境变量[${green(name)}][${count}]`, isSuccess ? '成功' : r); if (isSuccess && item) item.value = value; return value; diff --git a/src/lib/w2RulesManage.ts b/src/lib/w2RulesManage.ts index 858690e..728cbce 100644 --- a/src/lib/w2RulesManage.ts +++ b/src/lib/w2RulesManage.ts @@ -2,7 +2,7 @@ * @Author: renxia * @Date: 2024-01-22 14:00:13 * @LastEditors: renxia - * @LastEditTime: 2024-03-04 22:48:50 + * @LastEditTime: 2024-05-08 13:59:33 * @Description: */ import { existsSync, readFileSync } from 'node:fs'; @@ -11,6 +11,7 @@ import { color } from '@lzwme/fe-utils'; import { Watcher } from './watch'; import { logger } from './helper'; import type { WhistleRuleItem } from '../../typings'; +import { isMatch } from 'micromatch'; type W2RuleMapItem = { rules: string[]; values?: Record; config?: WhistleRuleItem }; @@ -121,11 +122,12 @@ export function loadW2Rules(whistleRules: WhistleRuleItem[], action: 'clear' | ' const cache = { w2Rules: { - value: '', + value: {} as Record, update: 0, }, }; -export function getW2Rules() { + +export function getW2Rules(req: Whistle.PluginRequest) { const now = Date.now(); // 30s 缓存 @@ -138,8 +140,28 @@ export function getW2Rules() { Object.assign(values, item.values); }); - cache.w2Rules.value = JSON.stringify({ rules: [...rulesSet].join('\n').replace(/#.+\n?/gm, ''), values }); + const rules = [...rulesSet] + .join('\n') + .split('\n') + .map(d => d.trim()) + .filter(d => d && d.startsWith('#') && d.includes(' ') && d.includes('//')); + cache.w2Rules.value = { rules, values }; + } + + const { rules, values } = cache.w2Rules.value; + const { fullUrl } = req; + + for (const line of rules) { + let isMatched = false; + let url: string | RegExp = line.split(' ')[0]; + if ((url as string).startsWith('^http')) url = new RegExp(url); + + if (url instanceof RegExp) isMatched = url.test(fullUrl); + else isMatched = fullUrl.includes(url) || isMatch(fullUrl, url); + + // 性能考虑，暂只支持一个规则 + if (isMatched) return { rules: [line], values }; } - return cache.w2Rules.value; + return; } diff --git a/src/rulesServer.ts b/src/rulesServer.ts index 8f83456..3af3dd5 100644 --- a/src/rulesServer.ts +++ b/src/rulesServer.ts @@ -2,7 +2,7 @@ * @Author: renxia * @Date: 2024-01-22 14:00:13 * @LastEditors: renxia - * @LastEditTime: 2024-03-01 16:28:09 + * @LastEditTime: 2024-05-08 14:01:31 * @Description: */ import { handlerW2RuleFiles, getW2Rules } from './lib/w2RulesManage'; @@ -15,6 +15,7 @@ export function rulesServer(server: Whistle.PluginServer, options: Whistle.Plugi await handlerW2RuleFiles({ path: isUrl ? '' : rulePath, url: isUrl ? rulePath : '' }); } - res.end(getW2Rules()); + const rules = getW2Rules(req); + rules ? res.end(rules) : res.end(); }); } diff --git a/src/server.ts b/src/server.ts index 03a8bbe..425e062 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,7 +9,7 @@ import { color } from '@lzwme/fe-utils'; import { toQueryString } from '@lzwme/fe-utils/cjs/common/url'; import { logger } from './lib/helper'; import { ruleHandler } from './lib/ruleHandler'; -import { rulesManage } from './lib/rulesManage'; +import { rulesManager } from './lib/rulesManager'; import * as util from './util/util'; export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) => { @@ -21,7 +21,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) => if (util.isRemote(req)) return req.passThrough(); const { method, headers, fullUrl: url } = req; - const reqHeaderRules = rulesManage.rules['req-header']; + const reqHeaderRules = rulesManager.rules['req-header']; logger.trace('[request]', color.cyan(method), color.gray(url)); if (reqHeaderRules?.size > 0) { @@ -40,7 +40,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) => async (body, next, ctx) => { reqBody = body; - const mockRules = rulesManage.rules['req-body']; + const mockRules = rulesManager.rules['req-body']; if (mockRules?.size > 0) { for (const rule of mockRules.values()) { try { @@ -64,7 +64,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) => next({ body }); }, async (body, next, ctx) => { - const resBodyRules = rulesManage.rules['res-body']; + const resBodyRules = rulesManager.rules['res-body']; if (resBodyRules?.size > 0) { for (const rule of resBodyRules.values()) { diff --git a/src/tunnelRulesServer.ts b/src/tunnelRulesServer.ts index 18ee762..60fd6f2 100644 --- a/src/tunnelRulesServer.ts +++ b/src/tunnelRulesServer.ts @@ -1,18 +1,22 @@ -import { getConfig } from './lib/getConfig'; -import { rulesManage } from './lib/rulesManage'; import { isMatch } from 'micromatch'; +import { getConfig } from './lib/getConfig'; +import { ruleMatcher } from './lib/ruleHandler'; +import { rulesManager } from './lib/rulesManager'; function mitmMatch(req: Whistle.PluginRequest) { - if (rulesManage.rules['res-body'].size === 0) return; + if (rulesManager.rules['res-body'].size === 0) return; const host = (req.headers.host || new URL(req.originalReq.fullUrl).host).split(':')[0]; - const resBodyRules = rulesManage.rules['res-body'].values(); + const resBodyRules = rulesManager.rules['res-body'].values(); - for (const item of resBodyRules) { - if (item.mitm) { - const ok = (item.mitm as (string | RegExp)[]).some(d => (d instanceof RegExp ? d.test(host) : isMatch(host, d))); + for (const rule of resBodyRules) { + if (rule.mitm) { + const ok = (rule.mitm as (string | RegExp)[]).some(d => (d instanceof RegExp ? d.test(host) : isMatch(host, d))); if (ok) return host; } + + const msg = ruleMatcher({ rule, req: req as unknown as Whistle.PluginServerRequest, res: null }); + if (!msg) return host; } } diff --git a/typings/index.d.ts b/typings/index.d.ts index 15c0d5d..2679a21 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -2,7 +2,7 @@ * @Author: renxia * @Date: 2024-01-11 16:53:50 * @LastEditors: renxia - * @LastEditTime: 2024-04-12 09:17:15 + * @LastEditTime: 2024-05-08 15:19:58 * @Description: */ /// @@ -104,8 +104,6 @@ export interface RuleItem { /** * MITM 域名匹配配置。 * 当 res-body 类型的规则命中时会主动启用 https 解析拦截(whistle 未启用 https 拦截时)。 - * 若 url 参数以 https:// 开头，则从其提取 host 域名配置部分作为 mitm 默认值。 - * 推荐配置该项以选择性的启用 https 拦截，以提升 whistle 代理性能与效率。 */ mitm?: string | RegExp | (string | RegExp)[]; /** url 匹配规则 */ @@ -129,7 +127,7 @@ export interface RuleItem { /** <${type}>handler 简写。根据 type 类型自动识别 */ handler?: (ctx: RuleHandlerParams) => PromiseMaybe; // /** 规则处理并返回环境变量配置。可以数组的形式返回多个 */ - // saveCookieHandler?: (ctx: RuleHandlerParams & { allCacheData: CacheData[] }) => PromiseMaybe; + // saveCookieHandler?: (ctx: RuleHandlerParams & { cacheData: CacheData[] }) => PromiseMaybe; // /** [mock] 接口模拟处理，返回需 mock 的结果。若返回为空则表示忽略 */ // mockHandler?: (ctx: RuleHandlerParams) => PromiseMaybe | Buffer | string | object>; // /** [modify] 接收到请求返回数据后修改或保存数据的处理 */ @@ -164,7 +162,12 @@ export type RuleHandlerParams = { /** req.headers.cookie 格式化为的对象格式 */ cookieObj: Record; /** 当设置了 getCacheUid 时，返回同一规则缓存的所有数据(包含由 getCacheUid 格式化返回的 data 数据) */ - allCacheData: CacheData[]; + cacheData: CacheData[]; + /** + * 同 cacheData + * @deprecated 将在后续版本废除，请使用 cacheData + */ + allCacheData?: CacheData[]; /** [on=req-body, res-body] 请求参数 body */ reqBody?: Record | Buffer; /** [on=res-body] 远程接口返回的 body */